diff --git a/netkan/netkan/metadata.py b/netkan/netkan/metadata.py index daac1a5..0a69e7a 100644 --- a/netkan/netkan/metadata.py +++ b/netkan/netkan/metadata.py @@ -96,23 +96,25 @@ def string_attrib(val: str) -> Dict[str, str]: 'StringValue': val, } - def sqs_message_attribs(self, high_ver: Optional['Ckan.Version'] = None) -> Dict[str, Any]: + def sqs_message_attribs(self, high_ver: Optional['Ckan.Version'] = None, high_ver_pre: Optional['Ckan.Version'] = None) -> Dict[str, Any]: attribs: Dict[str, Any] = { 'GameId': self.string_attrib(self.game_id or 'ksp') } if high_ver and not getattr(self, 'x_netkan_allow_out_of_order', False): attribs['HighestVersion'] = self.string_attrib(high_ver.string) + if high_ver_pre and not getattr(self, 'x_netkan_allow_out_of_order', False): + attribs['HighestVersionPrerelease'] = self.string_attrib(high_ver_pre.string) return attribs def sqs_message( - self, high_ver: Optional['Ckan.Version'] = None) -> SendMessageBatchRequestEntryTypeDef: + self, high_ver: Optional['Ckan.Version'] = None, high_ver_pre: Optional['Ckan.Version'] = None) -> SendMessageBatchRequestEntryTypeDef: hex_id = uuid.uuid4().hex return { 'Id': hex_id, 'MessageBody': self.contents, 'MessageGroupId': '1', 'MessageDeduplicationId': hex_id, - 'MessageAttributes': self.sqs_message_attribs(high_ver), + 'MessageAttributes': self.sqs_message_attribs(high_ver, high_ver_pre), } @@ -430,6 +432,9 @@ def licenses(self) -> List[str]: lic = self.license return lic if isinstance(lic, list) else [lic] + def is_prerelease(self) -> bool: + return self._raw.get('release_status') in ('testing', 'development') + @property def redistributable(self) -> bool: for lic in self.licenses(): diff --git a/netkan/netkan/repos.py b/netkan/netkan/repos.py index b0dc2cd..94c2323 100644 --- a/netkan/netkan/repos.py +++ b/netkan/netkan/repos.py @@ -195,9 +195,9 @@ def identifiers(self) -> Iterable[str]: and any(child.match('*.ckan') for child in path.iterdir())) - def all_latest_modules(self) -> Iterable[Ckan]: + def all_latest_modules(self, prerelease: bool = False) -> Iterable[Ckan]: return filter(None, - (self.highest_version_module(identifier) + (self.highest_version_module(identifier, prerelease) for identifier in self.identifiers())) def mod_path(self, identifier: str) -> Path: @@ -206,11 +206,16 @@ def mod_path(self, identifier: str) -> Path: def ckans(self, identifier: str) -> Iterable[Ckan]: return (Ckan(f) for f in self.mod_path(identifier).glob(self.CKANMETA_GLOB)) - def highest_version_module(self, identifier: str) -> Optional[Ckan]: - return max(self.ckans(identifier), + def highest_version_module(self, identifier: str, prerelease: bool) -> Optional[Ckan]: + return max((ck for ck in self.ckans(identifier) + if ck.is_prerelease == prerelease), default=None, key=lambda ck: ck.version if ck else Ckan.Version('0')) def highest_version(self, identifier: str) -> Optional[Ckan.Version]: - highest = self.highest_version_module(identifier) + highest = self.highest_version_module(identifier, False) + return highest.version if highest else None + + def highest_version_prerelease(self, identifier: str) -> Optional[Ckan.Version]: + highest = self.highest_version_module(identifier, True) return highest.version if highest else None diff --git a/netkan/netkan/scheduler.py b/netkan/netkan/scheduler.py index a7fe9a2..f53d411 100644 --- a/netkan/netkan/scheduler.py +++ b/netkan/netkan/scheduler.py @@ -54,7 +54,9 @@ def _in_group(self, netkan: Netkan) -> bool: return self.webhooks_group if netkan.hook_only() else self.nonhooks_group def schedule_all_netkans(self) -> None: - messages = (nk.sqs_message(self.ckm_repo.highest_version(nk.identifier)) + repo = self.ckm_repo + messages = (nk.sqs_message(repo.highest_version(nk.identifier), + repo.highest_version_prerelease(nk.identifier)) for nk in self.nk_repo.netkans() if self._in_group(nk)) for batch in sqs_batch_entries(messages): self.client.send_message_batch(**self.sqs_batch_attrs(batch)) diff --git a/netkan/netkan/webhooks/github_inflate.py b/netkan/netkan/webhooks/github_inflate.py index daf1743..b151643 100644 --- a/netkan/netkan/webhooks/github_inflate.py +++ b/netkan/netkan/webhooks/github_inflate.py @@ -64,7 +64,9 @@ def inflate(ids: Iterable[str], game_id: str) -> None: if game.netkan_repo.git_repo.working_dir: # Make sure our NetKAN and CKAN-meta repos are up to date pull_all(game.repos) - messages = (nk.sqs_message(game.ckanmeta_repo.highest_version(nk.identifier)) + repo = game.ckanmeta_repo + messages = (nk.sqs_message(repo.highest_version(nk.identifier), + repo.highest_version_prerelease(nk.identifier)) for nk in netkans(str(game.netkan_repo.git_repo.working_dir), ids, game_id)) for batch in sqs_batch_entries(messages): current_config.client.send_message_batch( diff --git a/netkan/netkan/webhooks/inflate.py b/netkan/netkan/webhooks/inflate.py index 848a422..b231fee 100644 --- a/netkan/netkan/webhooks/inflate.py +++ b/netkan/netkan/webhooks/inflate.py @@ -22,7 +22,9 @@ def inflate_hook(game_id: str) -> Tuple[str, int]: return 'An array of identifiers is required', 400 # Make sure our NetKAN and CKAN-meta repos are up to date pull_all(game.repos) - messages = (nk.sqs_message(game.ckanmeta_repo.highest_version(nk.identifier)) + repo = game.ckanmeta_repo + messages = (nk.sqs_message(repo.highest_version(nk.identifier), + repo.highest_version_prerelease(nk.identifier)) for nk in netkans(str(game.netkan_repo.git_repo.working_dir), ids, game_id=game_id)) for batch in sqs_batch_entries(messages): current_app.logger.info( diff --git a/netkan/netkan/webhooks/spacedock_inflate.py b/netkan/netkan/webhooks/spacedock_inflate.py index 445cec4..1fc618e 100644 --- a/netkan/netkan/webhooks/spacedock_inflate.py +++ b/netkan/netkan/webhooks/spacedock_inflate.py @@ -44,9 +44,10 @@ def inflate_hook(game_id: str) -> Tuple[str, int]: return '', 204 # Submit them to the queue - messages = (nk.sqs_message( - current_config.common.game(game_id).ckanmeta_repo.highest_version(nk.identifier)) - for nk in nks) + repo = current_config.common.game(game_id).ckanmeta_repo + messages = (nk.sqs_message(repo.highest_version(nk.identifier), + repo.highest_version_prerelease(nk.identifier)) + for nk in nks) for batch in sqs_batch_entries(messages): current_config.client.send_message_batch( QueueUrl=current_config.inflation_queue(game_id).url,