diff --git a/README.md b/README.md index 6d8efca..32f0937 100644 --- a/README.md +++ b/README.md @@ -252,9 +252,3 @@ geo2tz update 2023b the `update` command will download the timezone geojson zip and generate a version file in the `tzdata` directory, the version file is used to track the current version of the database. - -## Tests and known issues - -Automated tests are executed for coordinates with a precision between 1.11m (4 decimals) and 11.1m (5 decimals), the test file is `db/testdata/coordinates.json`. - -Some of the location are marked with `err` since they produce incorrect results, this is due to the fact that the timezone boundaries often overlaps. If you have additional test cases that you think are relevant, please open an issue or a PR. diff --git a/db/db.go b/db/db.go index 8ecbb78..e015c5b 100644 --- a/db/db.go +++ b/db/db.go @@ -4,7 +4,6 @@ import "errors" type TzDBIndex interface { Lookup(lat, lon float64) (string, error) - Size() int } var ( diff --git a/db/rtree.go b/db/rtree.go index 3a131a5..ff4174c 100644 --- a/db/rtree.go +++ b/db/rtree.go @@ -3,17 +3,18 @@ package db import ( "encoding/json" "errors" + "fmt" "io" "strings" - "github.com/klauspost/compress/zip" + "archive/zip" + "github.com/tidwall/rtree" ) type Geo2TzRTreeIndex struct { land rtree.RTreeG[timezoneGeo] sea rtree.RTreeG[timezoneGeo] - size int } // IsOcean checks if the timezone is for oceans @@ -23,7 +24,6 @@ func IsOcean(label string) bool { // Insert adds a new timezone bounding box to the index func (g *Geo2TzRTreeIndex) Insert(min, max [2]float64, element timezoneGeo) { - g.size++ if IsOcean(element.Name) { g.sea.Insert(min, max, element) return @@ -31,6 +31,7 @@ func (g *Geo2TzRTreeIndex) Insert(min, max [2]float64, element timezoneGeo) { g.land.Insert(min, max, element) } +// NewGeo2TzRTreeIndexFromGeoJSON creates a new Geo2TzRTreeIndex from a GeoJSON file func NewGeo2TzRTreeIndexFromGeoJSON(geoJSONPath string) (*Geo2TzRTreeIndex, error) { // open the zip file zipFile, err := zip.OpenReader(geoJSONPath) @@ -43,9 +44,9 @@ func NewGeo2TzRTreeIndexFromGeoJSON(geoJSONPath string) (*Geo2TzRTreeIndex, erro gri := &Geo2TzRTreeIndex{} // this function will add the timezone polygons to the shape index - iter := func(tz timezoneGeo) error { + iter := func(tz *timezoneGeo) error { for _, p := range tz.Polygons { - gri.Insert([2]float64{p.MinLat, p.MinLng}, [2]float64{p.MaxLat, p.MaxLng}, tz) + gri.Insert([2]float64{p.MinLat, p.MinLng}, [2]float64{p.MaxLat, p.MaxLng}, *tz) } return nil } @@ -114,10 +115,7 @@ func (g *Geo2TzRTreeIndex) Lookup(lat, lng float64) (tzID string, err error) { return } -func (g *Geo2TzRTreeIndex) Size() int { - return g.size -} - +// isPointInPolygonPIP checks if a point is inside a polygon using the Point in Polygon algorithm func isPointInPolygonPIP(point vertex, polygon polygon) bool { oddNodes := false n := len(polygon.Vertices) @@ -139,8 +137,10 @@ func isPointInPolygonPIP(point vertex, polygon polygon) bool { GeoJSON processing */ -// Polygon represents a polygon -// with a list of vertices [lat, lng] +type timezoneGeo struct { + Name string + Polygons []polygon +} type polygon struct { Vertices []vertex MaxLat float64 @@ -149,50 +149,39 @@ type polygon struct { MinLng float64 } -type vertex struct { - lat, lng float64 +func newPolygon() polygon { + return polygon{ + Vertices: make([]vertex, 0), + MaxLat: -90, + MinLat: 90, + MaxLng: -180, + MinLng: 180, + } } -type GeoJSONFeature struct { - Type string `json:"type"` - Properties struct { - TzID string `json:"tzid"` - } `json:"properties"` - Geometry struct { - Item string `json:"type"` - Coordinates []interface{} `json:"coordinates"` - } `json:"geometry"` +type vertex struct { + lat, lng float64 } func (p *polygon) AddVertex(lat, lng float64) { - if len(p.Vertices) == 0 { + + if lat > p.MaxLat { p.MaxLat = lat + } + if lat < p.MinLat { p.MinLat = lat + } + if lng > p.MaxLng { p.MaxLng = lng + } + if lng < p.MinLng { p.MinLng = lng - } else { - if lat > p.MaxLat { - p.MaxLat = lat - } - if lat < p.MinLat { - p.MinLat = lat - } - if lng > p.MaxLng { - p.MaxLng = lng - } - if lng < p.MinLng { - p.MinLng = lng - } } - p.Vertices = append(p.Vertices, vertex{lat, lng}) -} -type timezoneGeo struct { - Name string - Polygons []polygon + p.Vertices = append(p.Vertices, vertex{lat, lng}) } -func decodeJSON(f *zip.File, iter func(tz timezoneGeo) error) (err error) { +func decodeJSON(f *zip.File, iter func(tz *timezoneGeo) error) (err error) { var rc io.ReadCloser if rc, err = f.Open(); err != nil { return err @@ -215,61 +204,73 @@ func decodeJSON(f *zip.File, iter func(tz timezoneGeo) error) (err error) { return errors.New("error no features found") } -func decodeFeatures(dec *json.Decoder, fn func(tz timezoneGeo) error) error { - var f GeoJSONFeature +func decodeFeatures(dec *json.Decoder, fn func(tz *timezoneGeo) error) error { var err error + toPolygon := func(raw any) (polygon, error) { + container, ok := raw.([]any) + if !ok { + return polygon{}, fmt.Errorf("invalid polygon data, expected[][]any, got %T", raw) + } + + p := newPolygon() + for _, c := range container { + c, ok := c.([]any) + if !ok { + return p, fmt.Errorf("invalid container data, expected []any, got %T", c) + } + if len(c) != 2 { + return p, fmt.Errorf("invalid point data, expected 2, got %v", len(c)) + } + lat, ok := c[1].(float64) + if !ok { + return p, fmt.Errorf("invalid lat data, float64, got %T", c) + } + lng, ok := c[0].(float64) + if !ok { + return p, fmt.Errorf("invalid lng data, float64, got %T", c) + } + p.AddVertex(lat, lng) + } + return p, nil + } + + var f struct { + Type string `json:"type"` + Properties struct { + TzID string `json:"tzid"` + } `json:"properties"` + Geometry struct { + Item string `json:"type"` + Coordinates []any `json:"coordinates"` + } `json:"geometry"` + } for dec.More() { if err = dec.Decode(&f); err != nil { return err } - var pp []polygon + tg := &timezoneGeo{Name: f.Properties.TzID} switch f.Geometry.Item { case "Polygon": - pp = decodePolygons(f.Geometry.Coordinates) + // we ignore the holes, that is why we only take the first block of coordinates + p, err := toPolygon(f.Geometry.Coordinates[0]) + if err != nil { + return err + } + tg.Polygons = []polygon{p} case "MultiPolygon": - pp = decodeMultiPolygons(f.Geometry.Coordinates) + for _, multi := range f.Geometry.Coordinates { + // we ignore the holes, that is why we only take the first block of coordinates + p, err := toPolygon(multi.([]any)[0]) + if err != nil { + return err + } + tg.Polygons = append(tg.Polygons, p) + } } - if err = fn(timezoneGeo{Name: f.Properties.TzID, Polygons: pp}); err != nil { + if err = fn(tg); err != nil { return err } } - return nil } - -// decodePolygons -// GeoJSON Spec https://geojson.org/geojson-spec.html -// Coordinates: [Longitude, Latitude] -func decodePolygons(polygons []interface{}) []polygon { - var pp []polygon - for _, points := range polygons { - p := polygon{} - for _, i := range points.([]interface{}) { - if latlng, ok := i.([]interface{}); ok { - p.AddVertex(latlng[1].(float64), latlng[0].(float64)) - } - } - pp = append(pp, p) - } - return pp -} - -// decodeMultiPolygons -// GeoJSON Spec https://geojson.org/geojson-spec.html -// Coordinates: [Longitude, Latitude] -func decodeMultiPolygons(polygons []interface{}) []polygon { - var pp []polygon - for _, v := range polygons { - p := polygon{} - for _, points := range v.([]interface{}) { // 2 - for _, i := range points.([]interface{}) { - if latlng, ok := i.([]interface{}); ok { - p.AddVertex(latlng[1].(float64), latlng[0].(float64)) - } - } - } - pp = append(pp, p) - } - return pp -} diff --git a/db/rtree_test.go b/db/rtree_test.go index 5d9f0de..ba4f3ed 100644 --- a/db/rtree_test.go +++ b/db/rtree_test.go @@ -1,6 +1,7 @@ package db import ( + "archive/zip" "testing" "github.com/noandrea/geo2tz/v2/helpers" @@ -20,7 +21,6 @@ func TestGeo2TzTreeIndex_LookupZone(t *testing.T) { // load the database gsi, err := NewGeo2TzRTreeIndexFromGeoJSON("../tzdata/timezones.zip") assert.NoError(t, err) - assert.NotEmpty(t, gsi.Size()) // load the coordinates err = helpers.LoadJSON("testdata/coordinates.json", &tests) @@ -31,7 +31,7 @@ func TestGeo2TzTreeIndex_LookupZone(t *testing.T) { t.Run(tt.Tz, func(t *testing.T) { got, err := gsi.Lookup(tt.Lat, tt.Lon) if tt.NotFound { - assert.ErrorIs(t, err, ErrNotFound) + assert.ErrorIs(t, err, ErrNotFound, "expected %s to be not_found for https://www.google.com/maps/@%v,%v,12z", got, tt.Lat, tt.Lon) return } assert.NoError(t, err) @@ -45,7 +45,6 @@ func BenchmarkGeo2TzTreeIndex_LookupZone(b *testing.B) { // load the database gsi, err := NewGeo2TzRTreeIndexFromGeoJSON("../tzdata/timezones.zip") assert.NoError(b, err) - assert.NotEmpty(b, gsi.Size()) // load the coordinates var tests []struct { @@ -70,3 +69,45 @@ func BenchmarkGeo2TzTreeIndex_LookupZone(b *testing.B) { } } } + +func Test_decodeJSON(t *testing.T) { + + // lis of expected polygons with the number of vertices + expected := map[string][]int{ + "Africa/Bamako": {29290}, + "America/New_York": {459, 31606, 17}, + "Asia/Tokyo": {133, 129, 129, 139, 55, 127, 22, 17, 148, 18, 17, 162, 129, 129, 198, 424, 129, 33, 26, 92, 634, 754, 1019, 518, 149, 2408}, + "Australia/Sydney": {43621}, + "Europe/Berlin": {151051, 5, 20, 150, 60, 605, 341}, + "Europe/Rome": {114, 137, 217, 51, 567, 53273}, + } + + got := map[string][]int{} + + zipFile, err := zip.OpenReader("testdata/timezones.zip") + assert.NoError(t, err) + + iter := func(tz *timezoneGeo) error { + got[tz.Name] = []int{} + for _, p := range tz.Polygons { + got[tz.Name] = append(got[tz.Name], len(p.Vertices)) + } + return nil + } + + err = decodeJSON(zipFile.File[0], iter) + assert.NoError(t, err) + + for expectedTz, expectedPolySizes := range expected { + gotPolySizes, ok := got[expectedTz] + assert.True(t, ok, "expected %s to be in the got map", expectedTz) + areEq := assert.Equal(t, len(expectedPolySizes), len(gotPolySizes), "expected %s to have %d polygons, got %d", expectedTz, len(expectedPolySizes), len(gotPolySizes)) + + if !areEq { + continue + } + for i := range expectedPolySizes { + assert.Equal(t, expectedPolySizes[i], gotPolySizes[i], "expected %s to have polygon %d with %d vertices, got %d", expectedTz, i, expectedPolySizes[i], gotPolySizes[i]) + } + } +} diff --git a/db/testdata/coordinates.json b/db/testdata/coordinates.json index fcf5fa6..64e6fee 100755 --- a/db/testdata/coordinates.json +++ b/db/testdata/coordinates.json @@ -25,8 +25,7 @@ "lat": 43.42582, "lon": 11.831443, "tz": "Europe/Rome", - "note": "https://github.com/noandrea/geo2tz/issues/22", - "not_found": true + "note": "https://github.com/noandrea/geo2tz/issues/22" }, { "lat": 32.7767, "lon": -96.797, "tz": "America/Chicago" }, { "lat": 34.0522, "lon": -118.2437, "tz": "America/Los_Angeles" }, @@ -54,8 +53,7 @@ { "lat": 33.8688, "lon": 151.2093, - "tz": "Australia/Sydney", - "not_found": true, + "tz": "Etc/GMT-10", "note": "it's in the middle of the ocean" }, { "lat": 50.1109, "lon": 8.6821, "tz": "Europe/Berlin" }, diff --git a/db/testdata/timezones.zip b/db/testdata/timezones.zip new file mode 100644 index 0000000..981d512 Binary files /dev/null and b/db/testdata/timezones.zip differ diff --git a/go.mod b/go.mod index 59937d7..ed52639 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/noandrea/geo2tz/v2 go 1.21 require ( - github.com/klauspost/compress v1.17.9 github.com/labstack/echo/v4 v4.12.0 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 diff --git a/go.sum b/go.sum index 82dc6de..de10b58 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,6 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=