First data filtering api route (supports RBAC policy)

.dockerignore

@@ -2,3 +2,4 @@
 helm/
 .venv/
 .github/
+lib/

.gitignore

@@ -14,7 +14,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
 lib64/
 parts/
 sdist/

Makefile

@@ -24,7 +24,7 @@ build-arm64: prepare
 	@docker buildx build --platform linux/arm64 -t permitio/pdp-v2:$(VERSION) . --load
 
 run: run-prepare
-	@docker run -p 7766:7000 --env PDP_API_KEY=$(API_KEY) --env PDP_DEBUG=true permitio/pdp-v2:$(VERSION)
+	@docker run -it -p 7766:7000 -p 8181:8181 --env PDP_API_KEY=$(API_KEY) --env PDP_DEBUG=true permitio/pdp-v2:$(VERSION)
 
 run-on-background: run-prepare
-	@docker run -d -p 7766:7000 --env PDP_API_KEY=$(API_KEY) --env PDP_DEBUG=true permitio/pdp-v2:$(VERSION)
+	@docker run -d -p 7766:7000 -p 8181:8181 --env PDP_API_KEY=$(API_KEY) --env PDP_DEBUG=true permitio/pdp-v2:$(VERSION)

horizon/enforcer/api.py

@@ -4,7 +4,7 @@
 from typing import cast, Optional, Union, Dict, List
 
 import aiohttp
-from fastapi import APIRouter, Depends, Header
+from fastapi import APIRouter, Depends, Header, Query
 from fastapi import HTTPException
 from fastapi import Request, Response, status
 from opal_client.config import opal_client_config
@@ -14,6 +14,7 @@
     DEFAULT_POLICY_STORE_GETTER,
 )
 from opal_client.utils import proxy_response
+from permit_datafilter.compile_api.compile_client import OpaCompileClient
 from pydantic import parse_obj_as
 from starlette.responses import JSONResponse
 
@@ -57,6 +58,34 @@
 AUTHORIZED_USERS_POLICY_PACKAGE = "permit.authorized_users.authorized_users"
 USER_TENANTS_POLICY_PACKAGE = USER_PERMISSIONS_POLICY_PACKAGE + ".tenants"
 KONG_ROUTES_TABLE_FILE = "/config/kong_routes.json"
+MAIN_PARTIAL_EVAL_PACKAGE = "permit.partial_eval"
+
+# TODO: a more robust policy needs to be added to permit default managed policy
+# this policy is partial-eval friendly, but only supports RBAC at the moment
+TEMP_PARTIAL_EVAL_POLICY = """
+package permit.partial_eval
+
+import future.keywords.contains
+import future.keywords.if
+import future.keywords.in
+
+default allow := false
+
+allow if {
+	checked_permission := sprintf("%s:%s", [input.resource.type, input.action])
+
+	some granting_role, role_data in data.roles
+	some resource_type, actions in role_data.grants
+	granted_action := actions[_]
+	granted_permission := sprintf("%s:%s", [resource_type, granted_action])
+
+	some tenant, roles in data.users[input.user.key].roleAssignments
+	role := roles[_]
+	role == granting_role
+	checked_permission == granted_permission
+	input.resource.tenant == tenant
+}
+"""
 
 stats_manager = StatisticsManager(
     interval_seconds=sidecar_config.OPA_CLIENT_FAILURE_THRESHOLD_INTERVAL,
@@ -676,4 +705,42 @@ async def is_allowed_kong(request: Request, query: KongAuthorizationQuery):
             )
         return result
 
+    @router.post(
+        "/filter_resources",
+        response_model=AuthorizationResult,
+        status_code=status.HTTP_200_OK,
+        response_model_exclude_none=True,
+        dependencies=[Depends(enforce_pdp_token)],
+    )
+    async def filter_resources(
+        request: Request,
+        input: AuthorizationQuery,
+        raw: bool = Query(
+            False,
+            description="whether we should include the OPA raw compilation result in the response. this can help us debug the translation of the AST",
+        ),
+        x_permit_sdk_language: Optional[str] = Depends(notify_seen_sdk),
+    ):
+        headers = transform_headers(request)
+        client = OpaCompileClient(
+            base_url=f"{opal_client_config.POLICY_STORE_URL}", headers=headers
+        )
+        COMPILE_ROOT_RULE_REFERENCE = f"data.{MAIN_PARTIAL_EVAL_PACKAGE}.allow"
+        query = f"{COMPILE_ROOT_RULE_REFERENCE} == true"
+        residual_policy = await client.compile_query(
+            query=query,
+            input=input.dict(),
+            unknowns=[
+                "input.resource.key",
+                "input.resource.tenant",
+                "input.resource.attributes",
+            ],
+            raw=raw,
+        )
+        return Response(
+            content=json.dumps(residual_policy.dict()),
+            status_code=status.HTTP_200_OK,
+            media_type="application/json",
+        )
+
     return router

horizon/state.py

@@ -125,7 +125,7 @@ def _get_pdp_version(cls) -> Optional[str]:
         if os.path.exists(PDP_VERSION_FILENAME):
             with open(PDP_VERSION_FILENAME) as f:
                 return f.read().strip()
-        return None
+        return "0.0.0"
 
     @classmethod
     def _get_pdp_runtime(cls) -> dict:

lib/permit-datafilter/MANIFEST.in

@@ -0,0 +1 @@
+include *.md requirements.txt

lib/permit-datafilter/Makefile

@@ -0,0 +1,11 @@
+.PHONY: help
+
+.DEFAULT_GOAL := help
+
+clean:
+	rm -rf *.egg-info build/ dist/
+
+publish:
+	$(MAKE) clean
+	python setup.py sdist bdist_wheel
+	python -m twine upload dist/*

lib/permit-datafilter/README.md

@@ -0,0 +1,9 @@
+# Permit.io Data Filtering SDK (EAP)
+
+Initial SDK to enable data filtering scenarios based on compiling OPA policies from Rego AST into SQL-like expressions.
+
+## Installation
+
+```py
+pip install permit-datafilter
+```

lib/permit-datafilter/permit_datafilter/boolean_expression/schemas.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Optional, Union
+
+from pydantic import BaseModel, Field, root_validator
+
+LOGICAL_AND = "and"
+LOGICAL_OR = "or"
+LOGICAL_NOT = "not"
+CALL_OPERATOR = "call"
+
+
+class BaseSchema(BaseModel):
+    class Config:
+        orm_mode = True
+        allow_population_by_field_name = True
+
+
+class Variable(BaseSchema):
+    """
+    represents a variable in a boolean expression tree.
+
+    if the boolean expression originated in OPA - this was originally a reference in the OPA document tree.
+    """
+
+    variable: str = Field(..., description="a path to a variable (reference)")
+
+
+class Value(BaseSchema):
+    """
+    Represents a value (literal) in a boolean expression tree.
+    Could be of any jsonable type: string, int, boolean, float, list, dict.
+    """
+
+    value: Any = Field(
+        ..., description="a literal value, typically compared to a variable"
+    )
+
+
+class Expr(BaseSchema):
+    operator: str = Field(..., description="the name of the operator")
+    operands: list["Operand"] = Field(..., description="the operands to the expression")
+
+
+class Expression(BaseSchema):
+    """
+    represents a boolean expression, comparised of logical operators (e.g: and/or/not) and comparison operators (e.g: ==, >, <)
+
+    we translate OPA call terms to expressions, treating the operator as the function name and the operands as the args.
+    """
+
+    expression: Expr = Field(
+        ...,
+        description="represents a boolean expression, comparised of logical operators (e.g: and/or/not) and comparison operators (e.g: ==, >, <)",
+    )
+
+
+Operand = Union[Variable, Value, "Expression"]
+
+
+class ResidualPolicyType(str, Enum):
+    ALWAYS_ALLOW = "always_allow"
+    ALWAYS_DENY = "always_deny"
+    CONDITIONAL = "conditional"
+
+
+class ResidualPolicyResponse(BaseSchema):
+    type: ResidualPolicyType = Field(..., description="the type of the residual policy")
+    condition: Optional["Expression"] = Field(
+        None,
+        description="an optional condition, exists if the type of the residual policy is CONDITIONAL",
+    )
+    raw: Optional[dict] = Field(
+        None,
+        description="raw OPA compilation result, provided for debugging purposes",
+    )
+
+    @root_validator
+    def check_condition_exists_when_needed(cls, values: dict):
+        type, condition = values.get("type"), values.get("condition", None)
+        if (
+            type == ResidualPolicyType.ALWAYS_ALLOW
+            or type == ResidualPolicyType.ALWAYS_DENY
+        ) and condition is not None:
+            raise ValueError(
+                f"invalid residual policy: a condition exists but the type is not CONDITIONAL, instead: {type}"
+            )
+        if type == ResidualPolicyType.CONDITIONAL and condition is None:
+            raise ValueError(
+                f"invalid residual policy: type is CONDITIONAL, but no condition is provided"
+            )
+        return values
+
+
+Expr.update_forward_refs()
+Expression.update_forward_refs()
+ResidualPolicyResponse.update_forward_refs()

lib/permit-datafilter/permit_datafilter/boolean_expression/translator.py

@@ -0,0 +1,120 @@
+from permit_datafilter.rego_ast import parser as ast
+from permit_datafilter.boolean_expression.schemas import (
+    CALL_OPERATOR,
+    Operand,
+    ResidualPolicyResponse,
+    ResidualPolicyType,
+    Expression,
+    Expr,
+    Value,
+    Variable,
+    LOGICAL_AND,
+    LOGICAL_OR,
+)
+
+
+def translate_opa_queryset(queryset: ast.QuerySet) -> ResidualPolicyResponse:
+    """
+    translates the Rego AST into a generic residual policy constructed as a boolean expression.
+
+    this boolean expression can then be translated by plugins into various SQL and ORMs.
+    """
+    if queryset.always_false:
+        return ResidualPolicyResponse(type=ResidualPolicyType.ALWAYS_DENY)
+
+    if queryset.always_true:
+        return ResidualPolicyResponse(type=ResidualPolicyType.ALWAYS_ALLOW)
+
+    if len(queryset.queries) == 1:
+        return ResidualPolicyResponse(
+            type=ResidualPolicyType.CONDITIONAL,
+            condition=translate_query(queryset.queries[0]),
+        )
+
+    queries = [query for query in queryset.queries if not query.always_true]
+
+    if len(queries) == 0:
+        # no not trival queries means always true
+        return ResidualPolicyResponse(type=ResidualPolicyType.ALWAYS_ALLOW)
+
+    # else, more than one query means there's a logical OR between queries
+    return ResidualPolicyResponse(
+        type=ResidualPolicyType.CONDITIONAL,
+        condition=Expression(
+            expression=Expr(
+                operator=LOGICAL_OR,
+                operands=[translate_query(query) for query in queries],
+            )
+        ),
+    )
+
+
+def translate_query(query: ast.Query) -> Expression:
+    if len(query.expressions) == 1:
+        return translate_expression(query.expressions[0])
+
+    return Expression(
+        expression=Expr(
+            operator=LOGICAL_AND,
+            operands=[
+                translate_expression(expression) for expression in query.expressions
+            ],
+        )
+    )
+
+
+def translate_expression(expression: ast.Expression) -> Expression:
+    if len(expression.terms) == 1 and expression.terms[0].type == ast.TermType.CALL:
+        # this is a call expression
+        return translate_call_term(expression.terms[0].value)
+
+    if not isinstance(expression.operator, ast.RefTerm):
+        raise ValueError(
+            f"The operator in an expression must be a term of type ref, instead got type {expression.operator.type} and value {expression.operator.value}"
+        )
+
+    return Expression(
+        expression=Expr(
+            operator=expression.operator.value.as_string,
+            operands=[translate_term(term) for term in expression.operands],
+        )
+    )
+
+
+def translate_call_term(call: ast.Call) -> Expression:
+    return Expression(
+        expression=Expr(
+            operator=CALL_OPERATOR,
+            operands=[
+                Expression(
+                    expression=Expr(
+                        operator=call.func,
+                        operands=[translate_term(term) for term in call.args],
+                    )
+                )
+            ],
+        )
+    )
+
+
+def translate_term(term: ast.Term) -> Operand:
+    if term.type in (
+        ast.TermType.NULL,
+        ast.TermType.BOOLEAN,
+        ast.TermType.NUMBER,
+        ast.TermType.STRING,
+    ):
+        return Value(value=term.value)
+
+    if term.type == ast.TermType.VAR:
+        return Variable(variable=term.value)
+
+    if term.type == ast.TermType.REF and isinstance(term.value, ast.Ref):
+        return Variable(variable=term.value.as_string)
+
+    if term.type == ast.TermType.CALL and isinstance(term.value, ast.Call):
+        return translate_call_term(term.value)
+
+    raise ValueError(
+        f"unable to translate term with type {term.type} and value {term.value}"
+    )