Update PAT to use Python3.11 (#493)

* Update PAT to Python3.11

* Auto-format files

* Auto-format files

* Revert public_api_client Path change; fmt

* Move non-test code to utils module; update panther_core

* Small fixes

---------

Co-authored-by: panther-bot-automation <[email protected]>

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ jobs:
       - name: Setup Python
         uses: actions/[email protected]
         with:
-          python-version: 3.9.15
+          python-version: 3.11
       - name: Install pipenv
         run: make install-pipenv
       - name: Setup Virtual Environment

.github/workflows/fmt.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: 3.11
 
       - name: Install pipenv
         run: make install-pipenv

.github/workflows/test_release_publish.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.9'
+        python-version: '3.11'
 
     - name: Install pip and pipenv
       run: |

Makefile

@@ -31,7 +31,7 @@ reqs:
 lint: ## Lint panther_analysis_tool (mypy, bandit, pylint)
 	pipenv run mypy $(packages) --disallow-untyped-defs --ignore-missing-imports --warn-unused-ignores
 	pipenv run bandit -r $(packages)
-	pipenv run pylint $(packages) --disable=missing-docstring,bad-continuation,duplicate-code,W0511,R0912,too-many-lines,too-few-public-methods --max-line-length=140
+	pipenv run pylint $(packages) --disable=missing-docstring,duplicate-code,W0511,R0912,W4901,too-many-lines,too-few-public-methods --max-line-length=140
 
 .PHONY: venv
 venv: ## Install dependencies (including dev dependencies) using pipenv
@@ -49,11 +49,11 @@ install: ## Install dependencies (including dev dependencies) using pipenv
 
 .PHONY: test
 test: ## Run panther_analysis_tool tests
-	pipenv run nosetests -v --with-coverage --cover-package=panther_analysis_tool --cover-html --cover-html-dir=htmlcov
+	pipenv run nose2 -v --with-coverage --coverage=panther_analysis_tool --coverage-report=html
 
 .PHONY: test-fail-fast
 test-fail-fast: ## Run panther_analysis_tool tests, stopping as soon as a test fails
-	pipenv run nosetests -v --stop
+	pipenv run nose2 -v --stop
 
 .PHONY: coverage
 coverage: ## Open the coverage report generated by the test target

Pipfile

@@ -4,50 +4,54 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
-bandit = "==1.7.0"
-black = "==24.3.0"
+bandit = "==1.7.5"
+black = "==23.7.0"
+coverage = "==7.3.1"
 isort = "==5.12.0"
-mypy = "==0.812"
-pylint = "==2.7.2"
-pyfakefs = "==4.4.0"
-nose = "==1.3.7"
-PyYAML = "==6.0.0"
-coverage = "==6.4.2"
-twine = "==4.0.2"
-responses = "==0.25.0"
-typed-ast = { version = "==1.4.3", markers = "python_version < '3.8' and implementation_name == 'cpython'" }
+mypy = "==1.5.1"
+nose2 = ">=0.13.0"
+panther-core = "==0.10.0"
+pyfakefs = "==5.4.1"
+pylint = "==2.17.5"
+PyYAML = "==6.0.1"
+responses = "*"
+twine = "*"
+typed-ast = "==1.5.5"
+types-python-dateutil = "*"
+types-requests = "*"
 
 [packages]
-boto3 = "==1.21.42"
-jsonpath-ng = "==1.5.3"
-requests = "==2.31.0"
-schema = "==0.7.5"
-semver = "==2.13.0"
-gql = { extras = ["aiohttp"], version = "==3.5.0" }
-graphql-core = "==3.2.1"
-"ruamel.yaml" = "==0.17.21"
-botocore = "==1.24.42"
+"ruamel.yaml.clib" = "==0.2.7"
+"ruamel.yaml" = "==0.17.32"
+boto3 = "==1.28.44"
+botocore = "==1.31.44"
 certifi = "==2023.7.22"
-chardet = "==4.0.0"
-contextlib2 = "==0.6.0.post1"
+chardet = "==5.2.0"
+contextlib2 = "==21.6.0"
 decorator = "==5.1.1"
-dynaconf = "==3.1.8"
-idna = "==3.7"
-jmespath = "==1.0.0"
-jsonschema = "4.17.3"
-packaging = "==23.0"
+dill = "==0.3.7"
+dynaconf = "==3.2.2"
+gql = { extras = ["aiohttp"], version = "==3.4.1" }
+graphql-core = "==3.2.3"
+idna = "==3.4"
+jmespath = "==1.0.1"
+jsonlines = "==4.0.0"
+jsonpath-ng = "==1.5.3"
+jsonschema = "4.19.0"
+nested-lookup = "==0.2.25"
+packaging = "==23.1"
 ply = "==3.11"
-policyuniverse = "==1.5.0.20220613"
+policyuniverse = "==1.5.1.20230817"
 python-dateutil = "==2.8.2"
-"ruamel.yaml.clib" = "==0.2.7"
-s3transfer = "==0.5.2"
+requests = "==2.31.0"
+s3transfer = "==0.6.2"
+schema = "==0.7.5"
+semver = "==2.13.0"
 six = "==1.16.0"
-sqlfluff = "==2.1.2"
-nested-lookup = "==0.2.25"
+sqlfluff = "==2.3.1"
+typing-extensions = "==4.7.1"
 urllib3 = "==1.26.18"
-typing-extensions = "==4.3.0"
-jsonlines = "==3.0.0"
-panther-core = "==0.8.1"
+wrapt = "==1.15.0"
 
 [requires]
-python_version = "3.9"
+python_version = "3.11"

panther_analysis_tool/analysis_utils.py

@@ -293,7 +293,7 @@ def __str__(self) -> str:
 
     def serialize_to_file(self) -> None:
         logging.debug("Writing analysis spec to %s", self.spec_filename)
-        with open(self.spec_filename, "w") as file:
+        with open(self.spec_filename, "w", encoding="utf-8") as file:
             self.yaml_ctx.dump(self.analysis_spec, file)
 
 
@@ -312,10 +312,10 @@ def get_yaml_loader(roundtrip: bool) -> YAML:
     # the YAML files.
     yaml = YAML(typ="rt")
     yaml.indent(mapping=2, sequence=4, offset=2)
-    yaml.preserve_quotes = True  # type: ignore
+    yaml.preserve_quotes = True
     yaml.default_flow_style = False
     # allow very long lines to avoid unnecessary line changes
-    yaml.width = 4096  # type: ignore
+    yaml.width = 4096
     return yaml
 
 
@@ -393,7 +393,7 @@ def load_analysis_specs_ex(
                 # setup yaml object
                 yaml = get_yaml_loader(roundtrip=roundtrip_yaml)
                 if fnmatch(filename, "*.y*ml"):
-                    with open(spec_filename, "r") as spec_file_obj:
+                    with open(spec_filename, "r", encoding="utf-8") as spec_file_obj:
                         try:
                             yield LoadAnalysisSpecsResult(
                                 spec_filename=spec_filename,
@@ -412,7 +412,7 @@ def load_analysis_specs_ex(
                                 error=err,
                             )
                 if fnmatch(filename, "*.json"):
-                    with open(spec_filename, "r") as spec_file_obj:
+                    with open(spec_filename, "r", encoding="utf-8") as spec_file_obj:
                         try:
                             yield LoadAnalysisSpecsResult(
                                 spec_filename=spec_filename,

panther_analysis_tool/backend/client.py

@@ -581,7 +581,7 @@ def backend_response_failed(resp: BackendResponse) -> bool:
 
 
 def to_bulk_upload_response(data: Any) -> BackendResponse[BulkUploadResponse]:
-    default_stats = dict(total=0, new=0, modified=0)
+    default_stats = {"total": 0, "new": 0, "modified": 0}
     return BackendResponse(
         status_code=200,
         data=BulkUploadResponse(

panther_analysis_tool/backend/lambda_client.py

@@ -73,6 +73,7 @@ def decode_body(res: BackendResponse) -> typing.Any:
     try:
         return json.loads(res.data["body"])
     except json.decoder.JSONDecodeError as decode_error:
+        # pylint: disable=broad-exception-raised
         raise Exception(res.data["body"]) from decode_error
 
 
@@ -109,6 +110,7 @@ def check(self) -> BackendCheckResponse:
         return BackendCheckResponse(success=True, message="not implemented")
 
     def async_bulk_upload(self, params: BulkUploadParams) -> BackendResponse[BulkUploadResponse]:
+        # pylint: disable=broad-exception-raised
         raise BaseException("async uploads not supported with lambda client")
 
     def bulk_upload(self, params: BulkUploadParams) -> BackendResponse[BulkUploadResponse]:
@@ -138,6 +140,7 @@ def bulk_upload(self, params: BulkUploadParams) -> BackendResponse[BulkUploadRes
         return to_bulk_upload_response(body)
 
     def bulk_validate(self, params: BulkUploadParams) -> BulkUploadValidateStatusResponse:
+        # pylint: disable=broad-exception-raised
         raise BaseException("bulk validate is not supported with lambda client")
 
     def delete_detections(
@@ -282,21 +285,25 @@ def update_schema(self, params: UpdateSchemaParams) -> BackendResponse:
     def transpile_simple_detection_to_python(
         self, params: TranspileToPythonParams
     ) -> BackendResponse[TranspileToPythonResponse]:
+        # pylint: disable=broad-exception-raised
         raise BaseException(
             "transpile simple detections to python is not supported with lambda client"
         )
 
     def test_correlation_rule(
         self, params: TestCorrelationRuleParams
     ) -> BackendResponse[TestCorrelationRuleResponse]:
+        # pylint: disable=broad-exception-raised
         raise BaseException("test correlation rule is not supported with lambda client")
 
     def transpile_filters(
         self, params: TranspileFiltersParams
     ) -> BackendResponse[TranspileFiltersResponse]:
+        # pylint: disable=broad-exception-raised
         raise BaseException("transpile filters is not supported with lambda client")
 
     def get_rule_body(self, params: GetRuleBodyParams) -> BackendResponse[GetRuleBodyResponse]:
+        # pylint: disable=broad-exception-raised
         raise BaseException("get rule body is not supported with lambda client")
 
     @staticmethod
@@ -332,9 +339,11 @@ def supports_perf_test(self) -> bool:
         return False
 
     def get_metrics(self, params: MetricsParams) -> BackendResponse[MetricsResponse]:
+        # pylint: disable=broad-exception-raised
         raise BaseException("get metrics is not supported with lambda client")
 
     def run_perf_test(self, params: PerfTestParams) -> BackendResponse[ReplayResponse]:
+        # pylint: disable=broad-exception-raised
         raise BaseException("run perf test is not supported with lambda client")
 
     def supports_enrich_test_data(self) -> bool:
@@ -343,7 +352,9 @@ def supports_enrich_test_data(self) -> bool:
     def generate_enriched_event_input(
         self, params: GenerateEnrichedEventParams
     ) -> BackendResponse[GenerateEnrichedEventResponse]:
+        # pylint: disable=broad-exception-raised
         raise BaseException("enrich-test-data is not supported with lambda client")
 
     def feature_flags(self, params: FeatureFlagsParams) -> BackendResponse[FeatureFlagsResponse]:
+        # pylint: disable=broad-exception-raised
         raise BaseException("feature-flags is not supported with lambda client")

panther_analysis_tool/backend/mocks.py

@@ -32,70 +32,70 @@
 
 
 class MockBackend(BackendClient):
-    def async_bulk_upload(self, params: BulkUploadParams) -> BackendResponse[BulkUploadResponse]:
+    def async_bulk_upload(self, params: BulkUploadParams) -> BackendResponse[BulkUploadResponse]:  # type: ignore
         pass
 
-    def bulk_upload(self, params: BulkUploadParams) -> BackendResponse[BulkUploadResponse]:
+    def bulk_upload(self, params: BulkUploadParams) -> BackendResponse[BulkUploadResponse]:  # type: ignore
         pass
 
-    def check(self) -> BackendCheckResponse:
+    def check(self) -> BackendCheckResponse:  # type: ignore
         pass
 
-    def list_schemas(self, params: ListSchemasParams) -> BackendResponse[Any]:
+    def list_schemas(self, params: ListSchemasParams) -> BackendResponse[Any]:  # type: ignore
         pass
 
-    def update_schema(self, params: UpdateSchemaParams) -> BackendResponse[Any]:
+    def update_schema(self, params: UpdateSchemaParams) -> BackendResponse[Any]:  # type: ignore
         pass
 
-    def delete_saved_queries(self, params: DeleteSavedQueriesParams) -> BackendResponse[Any]:
+    def delete_saved_queries(self, params: DeleteSavedQueriesParams) -> BackendResponse[Any]:  # type: ignore
         pass
 
-    def delete_detections(self, params: DeleteDetectionsParams) -> BackendResponse[Any]:
+    def delete_detections(self, params: DeleteDetectionsParams) -> BackendResponse[Any]:  # type: ignore
         pass
 
-    def supports_async_uploads(self) -> bool:
+    def supports_async_uploads(self) -> bool:  # type: ignore
         pass
 
-    def get_rule_body(self, params: GetRuleBodyParams) -> BackendResponse[GetRuleBodyResponse]:
+    def get_rule_body(self, params: GetRuleBodyParams) -> BackendResponse[GetRuleBodyResponse]:  # type: ignore
         pass
 
-    def transpile_simple_detection_to_python(
+    def transpile_simple_detection_to_python(  # type: ignore
         self, params: TranspileToPythonParams
     ) -> BackendResponse[Any]:
         pass
 
-    def test_correlation_rule(
+    def test_correlation_rule(  # type: ignore
         self, params: TestCorrelationRuleParams
     ) -> BackendResponse[TestCorrelationRuleResponse]:
         pass
 
-    def transpile_filters(
+    def transpile_filters(  # type: ignore
         self, params: TranspileFiltersParams
     ) -> BackendResponse[TranspileFiltersResponse]:
         pass
 
-    def supports_bulk_validate(self) -> bool:
+    def supports_bulk_validate(self) -> bool:  # type: ignore
         pass
 
-    def bulk_validate(self, params: BulkUploadParams) -> BulkUploadValidateStatusResponse:
+    def bulk_validate(self, params: BulkUploadParams) -> BulkUploadValidateStatusResponse:  # type: ignore
         pass
 
-    def supports_perf_test(self) -> bool:
+    def supports_perf_test(self) -> bool:  # type: ignore
         pass
 
-    def get_metrics(self, params: MetricsParams) -> BackendResponse[MetricsResponse]:
+    def get_metrics(self, params: MetricsParams) -> BackendResponse[MetricsResponse]:  # type: ignore
         pass
 
-    def run_perf_test(self, params: PerfTestParams) -> BackendResponse[ReplayResponse]:
+    def run_perf_test(self, params: PerfTestParams) -> BackendResponse[ReplayResponse]:  # type: ignore
         pass
 
-    def supports_enrich_test_data(self) -> bool:
+    def supports_enrich_test_data(self) -> bool:  # type: ignore
         pass
 
-    def generate_enriched_event_input(
+    def generate_enriched_event_input(  # type: ignore
         self, params: GenerateEnrichedEventParams
     ) -> BackendResponse[GenerateEnrichedEventResponse]:
         pass
 
-    def feature_flags(self, params: FeatureFlagsParams) -> BackendResponse[FeatureFlagsResponse]:
+    def feature_flags(self, params: FeatureFlagsParams) -> BackendResponse[FeatureFlagsResponse]:  # type: ignore
         pass

panther_analysis_tool/backend/public_api_client.py

@@ -87,7 +87,7 @@ class PublicAPIRequests:  # pylint: disable=too-many-public-methods
     _cache: Dict[str, str]
 
     def __init__(self) -> None:
-        self._cache = dict()
+        self._cache = {}
 
     def version_query(self) -> DocumentNode:
         return self._load("get_version")
@@ -154,7 +154,9 @@ def feature_flags_query(self) -> DocumentNode:
 
     def _load(self, name: str) -> DocumentNode:
         if name not in self._cache:
-            self._cache[name] = Path(_get_graphql_content_filepath(name)).read_text()
+            self._cache[name] = Path(_get_graphql_content_filepath(name)).read_text(
+                encoding="utf-8"
+            )
 
         return gql(self._cache[name])
 
@@ -261,11 +263,11 @@ def get_rule_body(self, params: GetRuleBodyParams) -> BackendResponse[GetRuleBod
         for test in tests:
             out_mocks = []
             for mock in test.get("mocks") or []:
-                out_mock = dict()
+                out_mock = {}
                 out_mock["ObjectName"] = mock["objectName"]
                 out_mock["ReturnValue"] = mock["returnValue"]
                 out_mocks.append(out_mock)
-            out_test = dict()
+            out_test = {}
             out_test["ExpectedResult"] = test["expectedResult"]
             out_test["Name"] = test["name"]
             out_test["Log"] = json.loads(test["resource"])