Merge branch 'expand-geoip'

- Allow configurable TZ selector string, to allow for more variation in providers

src/modules/locale/CMakeLists.txt

@@ -6,7 +6,7 @@ endif()
 
 include_directories( ${PROJECT_BINARY_DIR}/src/libcalamaresui )
 
-set( geoip_src GeoIP.cpp GeoIPFreeGeoIP.cpp )
+set( geoip_src GeoIP.cpp GeoIPJSON.cpp )
 set( geoip_libs )
 
 find_package(Qt5 COMPONENTS Xml)

src/modules/locale/GeoIP.cpp

@@ -20,13 +20,21 @@
 
 #include "utils/Logger.h"
 
+GeoIP::GeoIP(const QString& e)
+    : m_element( e )
+{
+}
+
 GeoIP::~GeoIP()
 {
 }
 
 GeoIP::RegionZonePair
-GeoIP::splitTZString( const QString& timezoneString )
+GeoIP::splitTZString( const QString& tz )
 {
+    QString timezoneString( tz );
+    timezoneString.remove( '\\' );
+
     QStringList tzParts = timezoneString.split( '/', QString::SkipEmptyParts );
     if ( tzParts.size() >= 2 )
     {

src/modules/locale/GeoIP.h

@@ -32,8 +32,9 @@ class QByteArray;
  * and can handle the data returned from its interpretation of that
  * configured URL, returning a region and zone.
  */
-struct GeoIP
+class GeoIP
 {
+public:
     using RegionZonePair = QPair<QString, QString>;
 
     virtual ~GeoIP();
@@ -51,6 +52,11 @@ struct GeoIP
 
     /** @brief Splits a region/zone string into a pair. */
     static RegionZonePair splitTZString( const QString& s );
+
+protected:
+    GeoIP( const QString& e = QString() );
+
+    QString m_element;  // string for selecting from data
 } ;
 
 #endif

src/modules/locale/GeoIPJSON.cpp

@@ -17,7 +17,7 @@
  *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "GeoIPFreeGeoIP.h"
+#include "GeoIPJSON.h"
 
 #include "utils/Logger.h"
 #include "utils/YamlUtils.h"
@@ -26,8 +26,13 @@
 
 #include <yaml-cpp/yaml.h>
 
+GeoIPJSON::GeoIPJSON(const QString& attribute)
+    : GeoIP( attribute.isEmpty() ? QLatin1String( "time_zone" ) : attribute )
+{
+}
+
 GeoIP::RegionZonePair
-FreeGeoIP::processReply( const QByteArray& data )
+GeoIPJSON::processReply( const QByteArray& data )
 {
     try
     {
@@ -39,12 +44,14 @@ FreeGeoIP::processReply( const QByteArray& data )
             var.type() == QVariant::Map )
         {
             QVariantMap map = var.toMap();
-            if ( map.contains( "time_zone" ) &&
-                !map.value( "time_zone" ).toString().isEmpty() )
+            if ( map.contains( m_element ) &&
+                !map.value( m_element ).toString().isEmpty() )
             {
-                return splitTZString( map.value( "time_zone" ).toString() );
+                return splitTZString( map.value( m_element ).toString() );
             }
         }
+        else
+            cWarning() << "Invalid YAML data for GeoIPJSON";
     }
     catch ( YAML::Exception& e )
     {

src/modules/locale/GeoIPJSON.h

@@ -16,21 +16,28 @@
  *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef GEOIPFREEGEOIP_H
-#define GEOIPFREEGEOIP_H
+#ifndef GEOIPJSON_H
+#define GEOIPJSON_H
 
 #include "GeoIP.h"
 
-/** @brief GeoIP lookup via freegeoip.com
+/** @brief GeoIP lookup for services that return JSON.
  *
  * This is the original implementation of GeoIP lookup,
- * using the FreeGeoIP service, or similar which returns
- * data in the same format.
+ * (e.g. using the FreeGeoIP.net service), or similar.
  *
  * The data is assumed to be in JSON format with a time_zone attribute.
  */
-struct FreeGeoIP : public GeoIP
+class GeoIPJSON : public GeoIP
 {
+public:
+    /** @brief Configure the attribute name which is selected.
+     *
+     * If an empty string is passed in (not a valid attribute name),
+     * then "time_zone" is used.
+     */
+    explicit GeoIPJSON( const QString& attribute = QString() );
+
     virtual RegionZonePair processReply( const QByteArray& );
 } ;
 

src/modules/locale/GeoIPTests.cpp

@@ -18,7 +18,7 @@
 
 #include "GeoIPTests.h"
 
-#include "GeoIPFreeGeoIP.h"
+#include "GeoIPJSON.h"
 #ifdef HAVE_XML
 #include "GeoIPXML.h"
 #endif
@@ -40,14 +40,14 @@ GeoIPTests::initTestCase()
 {
 }
 
+static const char json_data_attribute[] =
+    "{\"time_zone\":\"Europe/Amsterdam\"}";
+
 void
 GeoIPTests::testJSON()
 {
-    static const char data[] =
-        "{\"time_zone\":\"Europe/Amsterdam\"}";
-
-    FreeGeoIP handler;
-    auto tz = handler.processReply( data );
+    GeoIPJSON handler;
+    auto tz = handler.processReply( json_data_attribute );
 
     QCOMPARE( tz.first, QLatin1String( "Europe" ) );
     QCOMPARE( tz.second, QLatin1String( "Amsterdam" ) );
@@ -60,12 +60,24 @@ GeoIPTests::testJSON()
     QCOMPARE( tz.first, "America" );
 }
 
+void GeoIPTests::testJSONalt()
+{
+    GeoIPJSON handler( "zona_de_hora" );
+
+    auto tz = handler.processReply( json_data_attribute );
+    QCOMPARE( tz.first, QString() );  // Not found
+
+    tz = handler.processReply( "tarifa: 12\nzona_de_hora: Europe/Madrid" );
+    QCOMPARE( tz.first, QLatin1String( "Europe" ) );
+    QCOMPARE( tz.second, QLatin1String( "Madrid" ) );
+}
+
 void
 GeoIPTests::testJSONbad()
 {
     static const char data[] = "time_zone: 1";
 
-    FreeGeoIP handler;
+    GeoIPJSON handler;
     auto tz = handler.processReply( data );
 
     tz = handler.processReply( data );
@@ -76,13 +88,13 @@ GeoIPTests::testJSONbad()
 
     tz = handler.processReply( "<html><body>404 Forbidden</body></html>" );
     QCOMPARE( tz.first, QString() );
+
+    tz = handler.processReply( "{ time zone = 'America/LosAngeles'}" );
+    QCOMPARE( tz.first, QString() );
 }
 
 
-void
-GeoIPTests::testXML()
-{
-    static const char data[] =
+static const char xml_data_ubiquity[] =
         R"(<Response>
   <Ip>85.150.1.1</Ip>
   <Status>OK</Status>
@@ -99,9 +111,12 @@ GeoIPTests::testXML()
   <TimeZone>Europe/Amsterdam</TimeZone>
 </Response>)";
 
+void
+GeoIPTests::testXML()
+{
 #ifdef HAVE_XML
-    XMLGeoIP handler;
-    auto tz = handler.processReply( data );
+    GeoIPXML handler;
+    auto tz = handler.processReply( xml_data_ubiquity );
 
     QCOMPARE( tz.first, QLatin1String( "Europe" ) );
     QCOMPARE( tz.second, QLatin1String( "Amsterdam" ) );
@@ -115,7 +130,7 @@ GeoIPTests::testXML2()
         "<Response><TimeZone>America/North Dakota/Beulah</TimeZone></Response>";
 
 #ifdef HAVE_XML
-    XMLGeoIP handler;
+    GeoIPXML handler;
     auto tz = handler.processReply( data );
 
     QCOMPARE( tz.first, QLatin1String( "America" ) );
@@ -123,11 +138,23 @@ GeoIPTests::testXML2()
 #endif
 }
 
+
+void GeoIPTests::testXMLalt()
+{
+#ifdef HAvE_XML
+    GeoIPXML handler( "ZT" );
+
+    auto tz = handler.processReply( "<A><B/><C><ZT>Moon/Dark_side</ZT></C></A>" );
+    QCOMPARE( tz.first, QLatin1String( "Moon" ) );
+    QCOMPARE( tz.second, QLatin1String( "Dark_side" ) );
+#endif
+}
+
 void
 GeoIPTests::testXMLbad()
 {
 #ifdef HAVE_XML
-    XMLGeoIP handler;
+    GeoIPXML handler;
     auto tz = handler.processReply( "{time_zone: \"Europe/Paris\"}" );
     QCOMPARE( tz.first, QString() );
 
@@ -138,3 +165,27 @@ GeoIPTests::testXMLbad()
     QCOMPARE( tz.first, QString() );
 #endif
 }
+
+void GeoIPTests::testSplitTZ()
+{
+    auto tz = GeoIP::splitTZString( QLatin1String("Moon/Dark_side") );
+    QCOMPARE( tz.first, QLatin1String("Moon") );
+    QCOMPARE( tz.second, QLatin1String("Dark_side") );
+
+    // Some providers return weirdly escaped data
+    tz = GeoIP::splitTZString( QLatin1String("America\\/NewYork") );
+    QCOMPARE( tz.first, QLatin1String("America") );
+    QCOMPARE( tz.second, QLatin1String("NewYork") );  // That's not actually the zone name
+
+    // Check that bogus data fails
+    tz = GeoIP::splitTZString( QString() );
+    QCOMPARE( tz.first, QString() );
+
+    tz = GeoIP::splitTZString( QLatin1String("America.NewYork") );
+    QCOMPARE( tz.first, QString() );
+
+    // Check that three-level is split properly
+    tz = GeoIP::splitTZString( QLatin1String("America/North Dakota/Beulah") );
+    QCOMPARE( tz.first, QLatin1String("America") );
+    QCOMPARE( tz.second, QLatin1String("North Dakota/Beulah") );
+}

src/modules/locale/GeoIPTests.h

@@ -31,10 +31,13 @@ public:
 private Q_SLOTS:
     void initTestCase();
     void testJSON();
+    void testJSONalt();
     void testJSONbad();
     void testXML();
     void testXML2();
+    void testXMLalt();
     void testXMLbad();
+    void testSplitTZ();
 };
 
 #endif

src/modules/locale/GeoIPXML.cpp

@@ -23,8 +23,13 @@
 #include <QNetworkReply>
 #include <QtXml/QDomDocument>
 
+GeoIPXML::GeoIPXML( const QString& element )
+    : GeoIP( element.isEmpty() ? QLatin1String( "TimeZone" ) : element )
+{
+}
+
 GeoIP::RegionZonePair
-XMLGeoIP::processReply( const QByteArray& data )
+GeoIPXML::processReply( const QByteArray& data )
 {
     QString domError;
     int errorLine, errorColumn;
@@ -32,7 +37,7 @@ XMLGeoIP::processReply( const QByteArray& data )
     QDomDocument doc;
     if ( doc.setContent( data, false, &domError, &errorLine, &errorColumn ) )
     {
-        const auto tzElements = doc.elementsByTagName( "TimeZone" );
+        const auto tzElements = doc.elementsByTagName( m_element );
         cDebug() << "GeoIP found" << tzElements.length() << "elements";
         for ( int it = 0; it < tzElements.length(); ++it )
         {

src/modules/locale/GeoIPXML.h

@@ -28,8 +28,16 @@
  * element, which contains the text (string) for the region/zone. This
  * format is  expected by, e.g. the Ubiquity installer.
  */
-struct XMLGeoIP : public GeoIP
+class GeoIPXML : public GeoIP
 {
+public:
+    /** @brief Configure the element tag which is selected.
+     *
+     * If an empty string is passed in (not a valid element tag),
+     * then "TimeZone" is used.
+     */
+    explicit GeoIPXML( const QString& element = QString() );
+
     virtual RegionZonePair processReply( const QByteArray& );
 } ;
 

src/modules/locale/LocaleViewStep.cpp

@@ -20,7 +20,7 @@
 #include "LocaleViewStep.h"
 
 #include "GeoIP.h"
-#include "GeoIPFreeGeoIP.h"
+#include "GeoIPJSON.h"
 #ifdef HAVE_XML
 #include "GeoIPXML.h"
 #endif
@@ -124,16 +124,16 @@ LocaleViewStep::fetchGeoIpTimezone()
     if ( m_geoipStyle.isEmpty() || m_geoipStyle == "legacy" )
     {
         actualUrl.append( "/json/" );
-        handler = new FreeGeoIP;
+        handler = new GeoIPJSON( m_geoipSelector );
     }
     else if ( m_geoipStyle == "json" )
     {
-        handler = new FreeGeoIP;
+        handler = new GeoIPJSON( m_geoipSelector );
     }
 #if defined(HAVE_XML)
     else if ( m_geoipStyle == "xml" )
     {
-        handler = new XMLGeoIP;
+        handler = new GeoIPXML( m_geoipSelector );
     }
 #endif
     else
@@ -295,4 +295,5 @@ LocaleViewStep::setConfigurationMap( const QVariantMap& configurationMap )
     // Optional
     m_geoipUrl = CalamaresUtils::getString( configurationMap, "geoipUrl" );
     m_geoipStyle = CalamaresUtils::getString( configurationMap, "geoipStyle" );
+    m_geoipSelector = CalamaresUtils::getString( configurationMap, "geoipSelector" );
 }

src/modules/locale/LocaleViewStep.h

@@ -75,8 +75,10 @@ private:
 
     QPair< QString, QString > m_startingTimezone;
     QString m_localeGenPath;
-    QString m_geoipUrl;
-    QString m_geoipStyle;
+
+    QString m_geoipUrl;  // The URL, depening on style might be modified on lookup
+    QString m_geoipStyle;  // String selecting which kind of geoip data to expect
+    QString m_geoipSelector;  // String selecting data from the geoip lookup
 
     QList< Calamares::job_ptr > m_jobs;
 };

src/modules/locale/locale.conf

@@ -37,7 +37,9 @@ zone:                       "New_York"
 # the URL may be modified before use. The request should return
 # valid data in a suitable format, depending on geoipStyle;
 # generally this includes a string value with the timezone
-# in <region>/<zone> format.
+# in <region>/<zone> format. For services that return data which
+# does not follow the conventions of "suitable data" described
+# below, *geoIPSelector* may be used to pick different data.
 #
 # Note that this example URL works, but the service is shutting
 # down in June 2018.
@@ -68,3 +70,12 @@ zone:                       "New_York"
 # shutting down in June 2018. There are other providers with the same
 # format. XML format is provided for Ubiquity.
 #geoipStyle:    "legacy"
+
+# GeoIP selector. Leave commented out for the default selector
+# (which depends on the style: JSON uses "time_zone" and XML
+# uses TimeZone, for the FreeGeoIP-alike and the Ubiquity-alike
+# respectively). If the service configured via *geoipUrl* uses
+# a different attribute name (e.g. "timezone") in JSON or a
+# different element tag (e.g. "<Time_Zone>") in XML, set this
+# string to the name or tag to be used.
+#geoipSelector: ""

src/modules/locale/test_geoip.cpp

@@ -22,7 +22,7 @@
 
 #include <iostream>
 
-#include "GeoIPFreeGeoIP.h"
+#include "GeoIPJSON.h"
 #ifdef HAVE_XML
 #include "GeoIPXML.h"
 #endif
@@ -39,10 +39,10 @@ int main(int argc, char** argv)
 
     GeoIP* handler = nullptr;
     if ( QLatin1String( "json" ) == argv[1] )
-        handler = new FreeGeoIP;
+        handler = new GeoIPJSON;
 #ifdef HAVE_XML
     else if ( QLatin1String( "xml" ) == argv[1] )
-        handler = new XMLGeoIP;
+        handler = new GeoIPXML;
 #endif
 
     if ( !handler )