Merge pull request #255 from snowplow/not_use_list

New function not based on list endpoint

modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/Resolver.scala

@@ -38,6 +38,12 @@ final case class Resolver[F[_]](repos: List[Registry], cache: Option[ResolverCac
   private[client] val allRepos: NonEmptyList[Registry] =
     NonEmptyList[Registry](Registry.EmbeddedRegistry, repos)
 
+  private val allIgluCentral: Set[String] = repos.collect {
+    case Registry.Http(config, connection)
+        if connection.uri.getHost.matches(""".*\biglucentral\b.*""") =>
+      config.name
+  }.toSet
+
   /**
    * Tries to find the given schema in any of the provided repository refs
    * If any of repositories gives non-non-found error, lookup will retried
@@ -181,6 +187,71 @@ final case class Resolver[F[_]](repos: List[Registry], cache: Option[ResolverCac
   ): F[Either[ResolutionError, Json]] =
     lookupSchemaResult(schemaKey).map(_.map(_.value.schema))
 
+  /**
+   * If Iglu Central or any of its mirrors doesn't have a schema,
+   * it should be considered NotFound, even if one of them returned an error.
+   */
+  private[resolver] def isNotFound(error: ResolutionError): Boolean = {
+    val (igluCentral, custom) = error.value.partition { case (repo, _) =>
+      allIgluCentral.contains(repo)
+    }
+    (igluCentral.isEmpty || igluCentral.values.exists(
+      _.errors.exists(_ == RegistryError.NotFound)
+    )) && custom.values.flatMap(_.errors).forall(_ == RegistryError.NotFound)
+  }
+
+  /**
+   * Looks up all the schemas with the same model until `maxSchemaKey`.
+   * For the schemas of previous revisions, it starts with addition = 0
+   * and increments it until a NotFound.
+   *
+   * @param maxSchemaKey The SchemaKey until which schemas of the same model should get returned
+   * @return All the schemas if all went well, [[Resolver.SchemaResolutionError]] with the first error that happened
+   *         while looking up the schemas if something went wrong.
+   */
+  def lookupSchemasUntil(
+    maxSchemaKey: SchemaKey
+  )(implicit
+    F: Monad[F],
+    L: RegistryLookup[F],
+    C: Clock[F]
+  ): F[Either[SchemaResolutionError, NonEmptyList[SelfDescribingSchema[Json]]]] = {
+    def go(
+      current: SchemaVer.Full,
+      acc: List[SelfDescribingSchema[Json]]
+    ): F[Either[SchemaResolutionError, NonEmptyList[SelfDescribingSchema[Json]]]] = {
+      val currentSchemaKey = maxSchemaKey.copy(version = current)
+      lookupSchema(currentSchemaKey).flatMap {
+        case Left(e) =>
+          if (current.addition === 0)
+            Monad[F].pure(Left(SchemaResolutionError(currentSchemaKey, e)))
+          else if (current.revision < maxSchemaKey.version.revision && isNotFound(e))
+            go(current.copy(revision = current.revision + 1, addition = 0), acc)
+          else
+            Monad[F].pure(Left(SchemaResolutionError(currentSchemaKey, e)))
+        case Right(json) =>
+          if (current.revision < maxSchemaKey.version.revision)
+            go(
+              current.copy(addition = current.addition + 1),
+              SelfDescribingSchema(SchemaMap(currentSchemaKey), json) :: acc
+            )
+          else if (current.addition < maxSchemaKey.version.addition)
+            go(
+              current.copy(addition = current.addition + 1),
+              SelfDescribingSchema(SchemaMap(currentSchemaKey), json) :: acc
+            )
+          else
+            Monad[F].pure(
+              Right(
+                NonEmptyList(SelfDescribingSchema(SchemaMap(currentSchemaKey), json), acc).reverse
+              )
+            )
+      }
+    }
+
+    go(SchemaVer.Full(maxSchemaKey.version.model, 0, 0), Nil)
+  }
+
   /**
    * Get list of available schemas for particular vendor and name part
    * Server supposed to return them in proper order
@@ -389,6 +460,8 @@ object Resolver {
    */
   case class SchemaItem(schema: Json, supersededBy: SupersededBy)
 
+  case class SchemaResolutionError(schemaKey: SchemaKey, error: ResolutionError)
+
   /** The result of doing a lookup with the resolver, carrying information on whether the cache was used */
   sealed trait ResolverResult[+K, +A] {
     def value: A

modules/core/src/test/resources/iglu-client-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/3-1-0

@@ -0,0 +1,23 @@
+{
+	"$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
+	"description": "Schema to test lookupSchemasUntil function",
+	"self": {
+		"vendor": "com.snowplowanalytics",
+		"name": "lookup-schemas-until",
+		"format": "jsonschema",
+		"version": "3-1-0"
+	},
+	"type": "object",
+	"properties": {
+		"field___1": {
+			"type": "string"
+		},
+		"field___2": {
+			"type": ["string", "null"]
+		}
+	},
+	"required": [
+		"field___1"
+	],
+	"additionalProperties": false
+}

modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-0-0

@@ -0,0 +1,20 @@
+{
+	"$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
+	"description": "Schema to test lookupSchemasUntil function",
+	"self": {
+		"vendor": "com.snowplowanalytics",
+		"name": "lookup-schemas-until",
+		"format": "jsonschema",
+		"version": "1-0-0"
+	},
+	"type": "object",
+	"properties": {
+		"field_1": {
+			"type": "string"
+		}
+	},
+	"required": [
+		"field_1"
+	],
+	"additionalProperties": false
+}

modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-1-0

@@ -0,0 +1,23 @@
+{
+	"$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
+	"description": "Schema to test lookupSchemasUntil function",
+	"self": {
+		"vendor": "com.snowplowanalytics",
+		"name": "lookup-schemas-until",
+		"format": "jsonschema",
+		"version": "1-1-0"
+	},
+	"type": "object",
+	"properties": {
+		"field_1": {
+			"type": "string"
+		},
+		"field_2": {
+			"type": ["string", "null"]
+		}
+	},
+	"required": [
+		"field_1"
+	],
+	"additionalProperties": false
+}

modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-1-1

@@ -0,0 +1,26 @@
+{
+	"$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
+	"description": "Schema to test lookupSchemasUntil function",
+	"self": {
+		"vendor": "com.snowplowanalytics",
+		"name": "lookup-schemas-until",
+		"format": "jsonschema",
+		"version": "1-1-1"
+	},
+	"type": "object",
+	"properties": {
+		"field_1": {
+			"type": "string"
+		},
+		"field_2": {
+			"type": ["string", "null"]
+		},
+		"field_3": {
+			"type": ["string", "null"]
+		}
+	},
+	"required": [
+		"field_1"
+	],
+	"additionalProperties": false
+}

modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-1-2

@@ -0,0 +1,29 @@
+{
+	"$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
+	"description": "Schema to test lookupSchemasUntil function",
+	"self": {
+		"vendor": "com.snowplowanalytics",
+		"name": "lookup-schemas-until",
+		"format": "jsonschema",
+		"version": "1-1-2"
+	},
+	"type": "object",
+	"properties": {
+		"field_1": {
+			"type": "string"
+		},
+		"field_2": {
+			"type": ["string", "null"]
+		},
+		"field_3": {
+			"type": ["string", "null"]
+		},
+		"field_4": {
+			"type": ["string", "null"]
+		}
+	},
+	"required": [
+		"field_1"
+	],
+	"additionalProperties": false
+}

modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-2-0

@@ -0,0 +1,32 @@
+{
+	"$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
+	"description": "Schema to test lookupSchemasUntil function",
+	"self": {
+		"vendor": "com.snowplowanalytics",
+		"name": "lookup-schemas-until",
+		"format": "jsonschema",
+		"version": "1-2-0"
+	},
+	"type": "object",
+	"properties": {
+		"field_1": {
+			"type": "string"
+		},
+		"field_2": {
+			"type": ["string", "null"]
+		},
+		"field_3": {
+			"type": ["string", "null"]
+		},
+		"field_4": {
+			"type": ["string", "null"]
+		},
+		"field_5": {
+			"type": ["string", "null"]
+		}
+	},
+	"required": [
+		"field_1"
+	],
+	"additionalProperties": false
+}

modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-2-1

@@ -0,0 +1,35 @@
+{
+	"$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
+	"description": "Schema to test lookupSchemasUntil function",
+	"self": {
+		"vendor": "com.snowplowanalytics",
+		"name": "lookup-schemas-until",
+		"format": "jsonschema",
+		"version": "1-2-1"
+	},
+	"type": "object",
+	"properties": {
+		"field_1": {
+			"type": "string"
+		},
+		"field_2": {
+			"type": ["string", "null"]
+		},
+		"field_3": {
+			"type": ["string", "null"]
+		},
+		"field_4": {
+			"type": ["string", "null"]
+		},
+		"field_5": {
+			"type": ["string", "null"]
+		},
+		"field_6": {
+			"type": ["string", "null"]
+		}
+	},
+	"required": [
+		"field_1"
+	],
+	"additionalProperties": false
+}

modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-2-2

@@ -0,0 +1,38 @@
+{
+	"$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
+	"description": "Schema to test lookupSchemasUntil function",
+	"self": {
+		"vendor": "com.snowplowanalytics",
+		"name": "lookup-schemas-until",
+		"format": "jsonschema",
+		"version": "1-2-2"
+	},
+	"type": "object",
+	"properties": {
+		"field_1": {
+			"type": "string"
+		},
+		"field_2": {
+			"type": ["string", "null"]
+		},
+		"field_3": {
+			"type": ["string", "null"]
+		},
+		"field_4": {
+			"type": ["string", "null"]
+		},
+		"field_5": {
+			"type": ["string", "null"]
+		},
+		"field_6": {
+			"type": ["string", "null"]
+		},
+		"field_7": {
+			"type": ["string", "null"]
+		}
+	},
+	"required": [
+		"field_1"
+	],
+	"additionalProperties": false
+}

modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-3-0

@@ -0,0 +1,41 @@
+{
+	"$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
+	"description": "Invalid schema to test lookupSchemasUntil function - Misses comma after properties",
+	"self": {
+		"vendor": "com.snowplowanalytics",
+		"name": "lookup-schemas-until",
+		"format": "jsonschema",
+		"version": "1-3-0"
+	},
+	"type": "object",
+	"properties": {
+		"field_1": {
+			"type": "string"
+		},
+		"field_2": {
+			"type": ["string", "null"]
+		},
+		"field_3": {
+			"type": ["string", "null"]
+		},
+		"field_4": {
+			"type": ["string", "null"]
+		},
+		"field_5": {
+			"type": ["string", "null"]
+		},
+		"field_6": {
+			"type": ["string", "null"]
+		},
+		"field_7": {
+			"type": ["string", "null"]
+		},
+		"field_8": {
+			"type": ["string", "null"]
+		}
+	}
+	"required": [
+		"field_1"
+	],
+	"additionalProperties": false
+}