/*
 * RiskScape™ Copyright New Zealand Institute for Earth Science Limited
 * (Earth Sciences New Zealand) is distributed for research purposes only
 * under the terms of AGPLv3.
 *
 * RiskScape™ Copyright 2026 New Zealand Institute for Earth Science
 * Limited (Earth Sciences New Zealand). All rights reserved. Source code
 * available under the AGPLv3.
 * 
 * This program is free software: you can redistribute it and/or modify it under
 *  the terms of the GNU Affero General Public License as published by the Free
 *  Software Foundation, either version 3 of the License, or (at your option) any
 *  later version.
 * 
 * This program is distributed for RESEARCH PURPOSES ONLY, in the hope that it will
 * be useful for research and education initiatives.
 * 
 * If you are not a researcher, or you are a researcher who wishes to use this
 * program on terms other than AGPLv3 (including those who wish to restrict the
 * distribution of any source code created using this program), please contact:
 * https://riskscape.org.nz
 * 
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Affero General Public License for more details.  You should have received a copy
 * of the GNU Affero General Public License along with this program.  If not, see
 * <http://www.gnu.org/licenses/>.
 * 
 * By way of summary only, under the AGPLv3:
 *     • Permissions of this strongest copyleft license are conditioned
 *       on making available complete source code of licensed works and
 *       modifications, which include larger works using a licensed work,
 *       under the same license.
 *     • Copyright and license notices must be preserved.
 *     • Contributors provide an express grant of patent rights.
 *     • When a modified version is used to provide a service over a
 *       network, the complete source code of the modified version must be made
 *       available.
 */
package nz.org.riskscape.engine.cli.tests;

import static nz.org.riskscape.engine.GeoHelper.*;

import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;

import org.geotools.referencing.CRS;
import org.junit.Test;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryCollection;
import org.locationtech.jts.geom.GeometryFactory;
import org.locationtech.jts.geom.LineString;
import org.locationtech.jts.geom.MultiLineString;
import org.locationtech.jts.geom.MultiPolygon;
import org.locationtech.jts.geom.Point;
import org.locationtech.jts.geom.Polygon;
import org.geotools.api.referencing.crs.CoordinateReferenceSystem;

import com.google.common.io.CharSource;
import com.google.common.io.Files;

import nz.org.riskscape.engine.FileSystemMatchers;
import nz.org.riskscape.engine.GeoHelper;
import nz.org.riskscape.engine.GeometryProblems;
import nz.org.riskscape.engine.Matchers;
import nz.org.riskscape.engine.Tuple;
import nz.org.riskscape.engine.data.Bookmark;
import nz.org.riskscape.engine.output.ShapefileWriter2;
import nz.org.riskscape.engine.pipeline.ExecutionContext;
import nz.org.riskscape.engine.relation.Relation;
import nz.org.riskscape.engine.types.Struct;
import nz.org.riskscape.engine.types.Struct.StructMember;
import nz.org.riskscape.engine.types.Types;
import nz.org.riskscape.problem.Problem;
import nz.org.riskscape.problem.ProblemSink;

public class ShapefileTest extends BaseModelRunCommandTest {

  @Override
  public void populateProjectAndSetupCommandsFromHome() {
    super.populateProjectAndSetupCommandsFromHome();
    evalCommand.pipelineFile = "-";
  }

  @Test
  public void aShapefileCanBeEchodOutInAVerySimplePipeline_GL307() throws Exception {
    // because of another bug, #308, we need a very simple pipeline to trigger #307
    setCommandInput("input('shapefile') as shapefile");

    evalCommand.run();

    assertThat(getTempDirectory(), FileSystemMatchers.hasFile(FileSystemMatchers.fileWithName("shapefile.shp")));
  }

  @Test
  public void willReadWithCorrectCharacterEncoding() throws Exception {
    setCommandInput("input('macrons.shp')"
        + " -> select({name}) "
        + " -> save(name: 'region-names', format: 'csv')");

    evalCommand.run();

    assertThat(getTempDirectory(), FileSystemMatchers.hasFile(FileSystemMatchers.fileWithName("region-names.csv")));

    List<List<String>> names = openCsv("region-names.csv", "name");

    assertThat(names, contains(
        contains(equalTo("Eketāhuna Kaikōura Puketūī Ōtāwēwē"))
    ));
  }

  @Test
  public void canReadShapefileAndReprojectTheGeometriesWithMissingDatumShiftParams() throws Exception {
    runCommand.modelId = "reproject";
    runCommand.run();

    assertThat(collectedSinkProblems, contains(
        Matchers.hasAncestorProblem(
            Matchers.isProblem(Problem.Severity.WARNING, GeometryProblems.class, "reprojectionIgnoringDatumShift"))
    ));

    assertThat(openCsv("results.csv", "the_geom"), contains(
        contains(
            GeoHelper.wktGeometryMatch("POINT (-40.61739603078453 175.37617537959557)", DEGREE_TOLERANCE_NEAREST_MM)
        )
    ));
  }

  @Test
  public void canReadShapefileWithCrsNameSetAndReprojectTheGeometries() throws Exception {
    runCommand.modelId = "reproject";
    // this bookmark has crs-name set so the full CRS definition is read including the datum shift params
    // note also that crs-longitude-first is not set, because shapefiles have this set by default now to
    // match the shapefile xy (long/lat) convention
    runCommand.parameters = Arrays.asList("bookmark='nz-points-shp-2193'");
    runCommand.run();

    // check there is no datum shift warnings
    assertThat(collectedSinkProblems, hasSize(0));

    assertThat(openCsv("results.csv", "the_geom"), contains(
        contains(
            GeoHelper.wktGeometryMatch("POINT (-40.61739607118445 175.37617537959557)", DEGREE_TOLERANCE_NEAREST_MM)
        )
    ));
  }

  @Test
  public void aShapefileCanBeEchodOutUsingNewerFormatAndReadBackIn() throws Exception {
    // the points.shp file has a variety of different data types, we read them in via the geotools shapefile reading
    // code, then write it out using the new format, read it back in and check the first (and only) row matches what we
    // expect
    setCommandInput("input('points') -> save(name: 'shapefile', format: 'shapefile')");

    evalCommand.run();

    assertThat(getTempDirectory(), FileSystemMatchers.hasFile(FileSystemMatchers.fileWithName("shapefile.shp")));
    assertThat(getTempDirectory(), FileSystemMatchers.hasFile(FileSystemMatchers.fileWithName("shapefile.prj")));
    CharSource chars =
        Files.asCharSource(getTempDirectory().resolve("shapefile.prj").toFile(), Charset.defaultCharset());

    CoordinateReferenceSystem parsedCrs = CRS.parseWKT(chars.read());
    CoordinateReferenceSystem expectedCRS = CRS.decode("EPSG:4326", true);
    assertTrue(CRS.equalsIgnoreMetadata(parsedCrs, expectedCRS));

    // check that a *.cpg file is created. We'll just check that it contains a valid charset.
    CharSource cpgChars =
        Files.asCharSource(getTempDirectory().resolve("shapefile.cpg").toFile(), Charset.defaultCharset());
    Charset charset = Charset.forName(cpgChars.read());
    assertNotNull(charset);

    String projectText = ""
        + "[project]\n"
        + "[bookmark output]\n"
        + "location = shapefile.shp\n";

    File newProjectFile = getTempDirectory().resolve("project.ini").toFile();
    Files.asCharSink(newProjectFile, Charset.defaultCharset()).write(projectText);

    populateProject(newProjectFile.toPath());

    ExecutionContext context = project.newExecutionContext();
    Relation relation = engine.getBookmarkResolvers().
        resolveAndValidate(project.getBookmarks().get("output"), context, Relation.class)
        .get().getData(Relation.class).get();

    Tuple tuple = relation.iterator().next();

    assertEquals(Long.valueOf(1L), tuple.fetch("id"));
    assertEquals(Long.valueOf(-44), tuple.fetch("short_int"));
    assertEquals(Long.valueOf(2147483647L), tuple.fetch("long_int"));
    assertEquals(Long.valueOf(999999999999999999L), tuple.fetch("vlong_int"));
    assertEquals(Double.valueOf(100000.000000000000000D), tuple.fetch("big_real"));
    assertEquals(Double.valueOf(-99999.99990D), tuple.fetch("short_real"));
    assertEquals("short", tuple.fetch("short_text"));
    assertEquals("asdfjasdlfkjasdklfjalskdfjaklsdfjaklsdjfklasdfjasdf", tuple.fetch("long_text"));
    Date date = tuple.fetch("date");
    LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate();
    assertEquals(LocalDate.of(2020, 11, 9), localDate);
  }

  @Test
  public void canWriteShapefileWithNullFirstGeom() throws Exception {
    doWriteShapefileWithNullFirstGeom("shapefile");
  }

  @Test
  public void canWriteShapefileWithNullFirstGeomGeoToolsWriter() throws Exception {
    doWriteShapefileWithNullFirstGeom("shapefile_geotools");
  }

  public void doWriteShapefileWithNullFirstGeom(String writerFormat) throws Exception {
    // the points.shp file has a variety of different data types, we read them in via the geotools shapefile reading
    // code, then write it out using the new format, read it back in and check the first (and only) row matches what we
    // expect
    setCommandInput(String.format("input('first-point-null') -> save(name: 'shapefile', format: '%s')", writerFormat));

    evalCommand.run();

    assertThat(getTempDirectory(), FileSystemMatchers.hasFile(FileSystemMatchers.fileWithName("shapefile.shp")));
    assertThat(getTempDirectory(), FileSystemMatchers.hasFile(FileSystemMatchers.fileWithName("shapefile.prj")));
    CharSource chars =
        Files.asCharSource(getTempDirectory().resolve("shapefile.prj").toFile(), Charset.defaultCharset());

    CoordinateReferenceSystem parsedCrs = CRS.parseWKT(chars.read());
    // shapefiles should be written in XY (aka forceXY=true)
    CoordinateReferenceSystem expectedCRS = CRS.decode("EPSG:2193", true);
    assertTrue(CRS.equalsIgnoreMetadata(parsedCrs, expectedCRS));

    String projectText = ""
        + "[project]\n"
        + "[bookmark output]\n"
        + "location = shapefile.shp\n"
        + "skip-invalid = true"; // need to skip the null geom entry

    File newProjectFile = getTempDirectory().resolve("project.ini").toFile();
    Files.asCharSink(newProjectFile, Charset.defaultCharset()).write(projectText);

    populateProject(newProjectFile.toPath());

    ExecutionContext context = project.newExecutionContext();
    Relation relation = engine.getBookmarkResolvers().
        resolveAndValidate(project.getBookmarks().get("output"), context, Relation.class)
        .get().getData(Relation.class).get();

    Tuple tuple = relation.iterator().next();

    assertEquals(Long.valueOf(2L), tuple.fetch("id"));
    assertEquals("point1", tuple.fetch("name"));
  }

  @Test
  public void canWriteNzPointInYX() throws Exception {
    doWriteNzPointInYX("shapefile");
  }

  @Test
  public void canWriteNzPointInYXGeotools() throws Exception {
    doWriteNzPointInYX("shapefile_geotools");
  }

  private void doWriteNzPointInYX(String writerFormat) throws Exception {
    // nz-points contains a point with YX axis order. The point is `POINT (5501000 1801000)`
    evalCommand.pipelineFile =
        String.format("input('nz-points') -> save(name: 'nz-points', format: '%s')", writerFormat);
    evalCommand.run();

    reset();

    // now lets read the shapefile we just wrote and save it back to CSV
    evalCommand.pipelineFile = String.format("input('%s') -> save(name: 'nz-points', format: 'csv')",
        getTempDirectory().resolve("nz-points.shp"));
    evalCommand.run();

    // we expect the result to be like nz-points.csv source file, but with the axis swapped around.
    List<List<String>> r = openCsv("nz-points.csv", "the_geom", "name");
    assertThat(r, contains(
        // Note the swapped axis
        Arrays.asList("POINT (1801000 5501000)", "one")
    ));
  }

  @Test
  public void nestedAttributesAreFlattenedIgnoringParentAttributeNames() throws Exception {
    // this tests the different attribute in the new shapefile writer (ShapefileWriter2)
    setCommandInput("input('shapefile') ->"
        + "select({foo: {id: id, name: desc, geom: the_geom}, *, bar: {name: desc}}) ->"
        + "save(name: 'shapefile', format: 'shapefile')");

    assertNull(evalCommand.run());

    assertThat(getTempDirectory(), FileSystemMatchers.hasFile(FileSystemMatchers.fileWithName("shapefile.shp")));

    String projectText = ""
        + "[project]\n"
        + "[bookmark output]\n"
        + "location = shapefile.shp\n";

    File newProjectFile = getTempDirectory().resolve("project.ini").toFile();
    Files.asCharSink(newProjectFile, Charset.defaultCharset()).write(projectText);

    populateProject(newProjectFile.toPath());

    ExecutionContext context = project.newExecutionContext();
    Relation relation = engine.getBookmarkResolvers().
        resolveAndValidate(project.getBookmarks().get("output"), context, Relation.class)
        .get().getData(Relation.class).get();

    Tuple tuple = relation.iterator().next();
    List<String> memberKeys =
        tuple.getStruct().getMembers().stream().map(StructMember::getKey).collect(Collectors.toList());

    assertEquals(
        Arrays.asList("the_geom", "id", "name", "the_geom_1", "id_1", "desc", "value", "name_1"),
        memberKeys
    );
  }

  /*
   * This probably belongs in the unit test, but it lacks the code for opening the file back up again.  I could move it
   * in to the same project as the shapefile reader, but that's a bit of a PITA right now and should probably happen as
   * part or a broader structural re-org
   */
  @Test
  public void newWriterCanWriteAndReadBackVariousGeometryTypes() throws Exception {
    GeometryFactory gf = project.getSridSet().getGeometryFactory(project.getDefaultCrs());

    writeAndReadBack(gf.createPoint(new Coordinate(1, 1)));
    writeAndReadBack(gf.createMultiPoint(new Point[] {
      gf.createPoint(new Coordinate(1, 1)),
      gf.createPoint(new Coordinate(1, 3))
    }));

    // shapefiles don't support individual line strings and polygons, they are always in a collection
    writeAndReadBack(
      gf.createLineString(new Coordinate[] {new Coordinate(0, 0), new Coordinate(1, 1)}),
      MultiLineString.class, mls -> mls.getGeometryN(0)
    );

    writeAndReadBack(
      gf.createMultiLineString(new LineString[] {
        gf.createLineString(new Coordinate[] {new Coordinate(0, 0), new Coordinate(1, 1)}),
        gf.createLineString(new Coordinate[] {new Coordinate(2, 2), new Coordinate(1, 1)})
      })
    );

    writeAndReadBack(
      gf.createPolygon(new Coordinate[] {
          new Coordinate(0, 0), new Coordinate(0, 2), new Coordinate(2, 2), new Coordinate(0, 0)
      }),
      MultiPolygon.class, mp -> mp.getGeometryN(0)
    );

    writeAndReadBack(
      gf.createMultiPolygon(new Polygon[] {
        gf.createPolygon(new Coordinate[] {
          new Coordinate(0, 0), new Coordinate(0, -2), new Coordinate(-2, -2), new Coordinate(0, 0)
        }),
        gf.createPolygon(new Coordinate[] {
          new Coordinate(10, 0), new Coordinate(10, 2), new Coordinate(12, 2), new Coordinate(10, 0)
        }),
        gf.createPolygon(new Coordinate[] {
          new Coordinate(1, 1), new Coordinate(1, 2), new Coordinate(0, 2), new Coordinate(1, 1)
        }),
      })
    );
  }

  private <T extends Geometry> void writeAndReadBack(Geometry toWrite) throws IOException {
    writeAndReadBack(toWrite, Geometry.class, Function.identity());
  }

  private <T extends Geometry> void writeAndReadBack(
      Geometry toWrite,
      Class<T> expectedType,
      Function<T, Geometry> extractor
  ) throws IOException {
    Struct type = Struct.of("id", Types.INTEGER, "geom", Types.GEOMETRY);
    File tmpFile = getTempDirectory().resolve("input.shp").toFile();
    // use a real writer
    ShapefileWriter2 writer = new ShapefileWriter2(tmpFile, type, project.getSridSet(), ProblemSink.DEVNULL);
    writer.write(Tuple.ofValues(type, 1, toWrite));
    writer.close();


    Bookmark bm = Bookmark.fromURI(tmpFile.toURI());
    Relation relation =
        engine.getBookmarkResolvers().resolveAndValidate(bm, project.newExecutionContext(), Relation.class).get().
        getData(Relation.class).get();

    Geometry read = relation.iterator().next().fetch("the_geom");

    if (expectedType.isInstance(read)) {
      read = extractor.apply(expectedType.cast(read));
      if (read instanceof GeometryCollection) {
        // they don't seem to be written in the same order as they are read back, so use set equality
        GeometryCollection readColl = (GeometryCollection) read;
        GeometryCollection toWriteColl = (GeometryCollection) toWrite;

        assertEquals(asSet(readColl), asSet(toWriteColl));

      } else {
        assertEquals(toWrite, read);
      }
    } else {
      fail("wrong geometry type, was " + read.getClass() + " but expected " + expectedType);
    }
  }

  private Set<Geometry> asSet(GeometryCollection collection) {
    Set<Geometry> set = new HashSet<>(collection.getNumGeometries());
    for (int i = 0; i < collection.getNumGeometries(); i++) {
      set.add(collection.getGeometryN(i));
    }
    return set;
  }

}
