extract datafiltering code into a separate package: permit-datafilter

.dockerignore

@@ -2,3 +2,4 @@
 helm/
 .venv/
 .github/
+lib/

.gitignore

@@ -14,7 +14,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
 lib64/
 parts/
 sdist/

horizon/enforcer/api.py

@@ -14,12 +14,12 @@
     DEFAULT_POLICY_STORE_GETTER,
 )
 from opal_client.utils import proxy_response
+from permit_datafilter.compile_api.compile_client import OpaCompileClient
 from pydantic import parse_obj_as
 from starlette.responses import JSONResponse
 
 from horizon.authentication import enforce_pdp_token
 from horizon.config import sidecar_config
-from horizon.enforcer.data_filtering.compile_api.compile_client import OpaCompileClient
 from horizon.enforcer.schemas import (
     AuthorizationQuery,
     AuthorizationResult,
@@ -722,12 +722,14 @@ async def filter_resources(
         x_permit_sdk_language: Optional[str] = Depends(notify_seen_sdk),
     ):
         headers = transform_headers(request)
-        client = OpaCompileClient(headers=headers)
+        client = OpaCompileClient(
+            base_url=f"{opal_client_config.POLICY_STORE_URL}", headers=headers
+        )
         COMPILE_ROOT_RULE_REFERENCE = f"data.{MAIN_PARTIAL_EVAL_PACKAGE}.allow"
         query = f"{COMPILE_ROOT_RULE_REFERENCE} == true"
         residual_policy = await client.compile_query(
             query=query,
-            input=input,
+            input=input.dict(),
             unknowns=[
                 "input.resource.key",
                 "input.resource.tenant",
@@ -741,34 +743,4 @@ async def filter_resources(
             media_type="application/json",
         )
 
-    @router.post(
-        "/filter_resources_get_sql",
-        response_model=AuthorizationResult,
-        status_code=status.HTTP_200_OK,
-        response_model_exclude_none=True,
-        dependencies=[Depends(enforce_pdp_token)],
-    )
-    async def filter_resources_get_sql(
-        request: Request,
-        input: AuthorizationQuery,
-        x_permit_sdk_language: Optional[str] = Depends(notify_seen_sdk),
-    ):
-        """
-        TODO: temp endpoint, instead we should wrap the capability in the SDK
-        """
-        headers = transform_headers(request)
-        client = OpaCompileClient(headers=headers)
-        COMPILE_ROOT_RULE_REFERENCE = f"data.{MAIN_PARTIAL_EVAL_PACKAGE}.allow"
-        query = f"{COMPILE_ROOT_RULE_REFERENCE} == true"
-        residual_policy = await client.compile_query(
-            query=query,
-            input=input,
-            unknowns=[
-                "input.resource.key",
-                "input.resource.tenant",
-                "input.resource.attributes",
-            ],
-            raw=True,
-        )
-
     return router

lib/permit-datafilter/MANIFEST.in

@@ -0,0 +1 @@
+include *.md requirements.txt

lib/permit-datafilter/Makefile

@@ -0,0 +1,11 @@
+.PHONY: help
+
+.DEFAULT_GOAL := help
+
+clean:
+	rm -rf *.egg-info build/ dist/
+
+publish:
+	$(MAKE) clean
+	python setup.py sdist bdist_wheel
+	python -m twine upload dist/*

lib/permit-datafilter/README.md

@@ -0,0 +1,9 @@
+# Permit.io Data Filtering SDK (EAP)
+
+Initial SDK to enable data filtering scenarios based on compiling OPA policies from Rego AST into SQL-like expressions.
+
+## Installation
+
+```py
+pip install permit-datafilter
+```

horizon/enforcer/data_filtering/boolean_expression/translator.py → lib/permit-datafilter/permit_datafilter/boolean_expression/translator.py

@@ -1,5 +1,5 @@
-from horizon.enforcer.data_filtering.rego_ast import parser as ast
-from horizon.enforcer.data_filtering.boolean_expression.schemas import (
+from permit_datafilter.rego_ast import parser as ast
+from permit_datafilter.boolean_expression.schemas import (
     CALL_OPERATOR,
     Operand,
     ResidualPolicyResponse,

horizon/enforcer/data_filtering/compile_api/compile_client.py → lib/permit-datafilter/permit_datafilter/compile_api/compile_client.py

@@ -1,24 +1,22 @@
 import json
 
 import aiohttp
-from fastapi import HTTPException, Response, status
-from opal_client.config import opal_client_config
-from opal_client.logger import logger
+from fastapi import HTTPException, status
+from loguru import logger
 
-from horizon.enforcer.schemas import AuthorizationQuery
-from horizon.enforcer.data_filtering.compile_api.schemas import CompileResponse
-from horizon.enforcer.data_filtering.rego_ast import parser as ast
-from horizon.enforcer.data_filtering.boolean_expression.schemas import (
+from permit_datafilter.compile_api.schemas import CompileResponse
+from permit_datafilter.rego_ast import parser as ast
+from permit_datafilter.boolean_expression.schemas import (
     ResidualPolicyResponse,
 )
-from horizon.enforcer.data_filtering.boolean_expression.translator import (
+from permit_datafilter.boolean_expression.translator import (
     translate_opa_queryset,
 )
 
 
 class OpaCompileClient:
-    def __init__(self, headers: dict):
-        self._base_url = f"{opal_client_config.POLICY_STORE_URL}"
+    def __init__(self, base_url: str, headers: dict):
+        self._base_url = base_url  # f"{opal_client_config.POLICY_STORE_URL}"
         self._headers = headers
         self._client = aiohttp.ClientSession(
             base_url=self._base_url, headers=self._headers
@@ -27,27 +25,27 @@ def __init__(self, headers: dict):
     async def compile_query(
         self,
         query: str,
-        input: AuthorizationQuery,
+        input: dict,
         unknowns: list[str],
         raw: bool = False,
     ) -> ResidualPolicyResponse:
         # we don't want debug rules when we try to reduce the policy into a partial policy
-        input = {**input.dict(), "use_debugger": False}
+        input = {**input, "use_debugger": False}
         data = {
             "query": query,
             "input": input,
             "unknowns": unknowns,
         }
         try:
-            logger.info("Compiling OPA query: {}", data)
+            logger.debug("Compiling OPA query: {}", data)
             async with self._client as session:
                 async with session.post(
                     "/v1/compile",
                     data=json.dumps(data),
                     raise_for_status=True,
                 ) as response:
                     opa_compile_result = await response.json()
-                    logger.info(
+                    logger.debug(
                         "OPA compile query result: status={status}, response={response}",
                         status=response.status,
                         response=json.dumps(opa_compile_result),

horizon/enforcer/data_filtering/rego_ast/parser.py → lib/permit-datafilter/permit_datafilter/rego_ast/parser.py

@@ -40,12 +40,11 @@
 from types import NoneType
 from typing import Generic, List, TypeVar
 
-from horizon.enforcer.data_filtering.compile_api.schemas import (
+from permit_datafilter.compile_api.schemas import (
     CRExpression,
     CRQuery,
     CompileResponse,
     CRTerm,
-    CRSupportModule,
 )
 
 

lib/permit-datafilter/permit_datafilter/sdk/permit_filter.py

@@ -0,0 +1,72 @@
+# from typing import Union
+
+# from horizon.enforcer import schemas as enforcer_schemas
+# from horizon.enforcer.api import MAIN_PARTIAL_EVAL_PACKAGE
+# from permit_datafilter.boolean_expression.schemas import (
+#     ResidualPolicyResponse,
+# )
+# from permit_datafilter.compile_api.compile_client import OpaCompileClient
+
+# User = Union[dict, str]
+# Action = str
+# Resource = Union[dict, str]
+
+
+# def normalize_user(user: User) -> dict:
+#     if isinstance(user, str):
+#         return dict(key=user)
+#     else:
+#         return user
+
+
+# def normalize_resource_type(resource: Resource) -> str:
+#     if isinstance(resource, dict):
+#         t = resource.get("type", None)
+#         if t is not None and isinstance(t, str):
+#             return t
+#         raise ValueError("no resource type provided")
+#     else:
+#         return resource
+
+
+# def filter_resource_query(
+#     user: User, action: Action, resource: Resource
+# ) -> enforcer_schemas.AuthorizationQuery:
+#     normalized_user = normalize_user(user)
+#     resource_type: str = normalize_resource_type(resource)
+#     return enforcer_schemas.AuthorizationQuery(
+#         user=normalized_user,
+#         action=action,
+#         resource=enforcer_schemas.Resource(type=resource_type),
+#     )
+
+
+# class Permit:
+#     """
+#     stub for future SDK code
+#     """
+
+#     def __init__(self, token: str):
+#         self._headers = {
+#             "Authorization": f"Bearer {token}",
+#             "Content-Type": "application/json",
+#         }
+
+#     async def filter_resources(
+#         self, user: User, action: Action, resource: Resource
+#     ) -> ResidualPolicyResponse:
+#         """
+#         stub for future permit.filter_resources() function
+#         """
+#         client = OpaCompileClient(headers=self._headers)
+#         input = filter_resource_query(user, action, resource)
+#         return await client.compile_query(
+#             query=f"data.{MAIN_PARTIAL_EVAL_PACKAGE}.allow == true",
+#             input=input,
+#             unknowns=[
+#                 "input.resource.key",
+#                 "input.resource.tenant",
+#                 "input.resource.attributes",
+#             ],
+#             raw=True,
+#         )

horizon/enforcer/data_filtering/sdk/permit_sqlalchemy.py → lib/permit-datafilter/permit_datafilter/sdk/permit_sqlalchemy.py

@@ -2,12 +2,11 @@
 
 import sqlalchemy as sa
 
-# import Column, Table, and_, not_, or_, select
 from sqlalchemy.orm import DeclarativeMeta, InstrumentedAttribute
 from sqlalchemy.sql import Select
 from sqlalchemy.sql.expression import BinaryExpression, ColumnOperators
 
-from horizon.enforcer.data_filtering.boolean_expression.schemas import (
+from permit_datafilter.boolean_expression.schemas import (
     CALL_OPERATOR,
     LOGICAL_AND,
     LOGICAL_NOT,

lib/permit-datafilter/requirements.txt

@@ -0,0 +1,7 @@
+aiohttp>=3.9.4,<4
+ddtrace
+fastapi>=0.109.1,<1
+loguru
+pydantic[email]>=1.9.1,<2
+pytest
+SQLAlchemy==1.4.46