Skip to content

Commit

Permalink
policyeval: actually evalute policies
Browse files Browse the repository at this point in the history
  • Loading branch information
tulir committed Sep 7, 2024
1 parent 44331ac commit 4e6fd57
Show file tree
Hide file tree
Showing 16 changed files with 498 additions and 102 deletions.
9 changes: 7 additions & 2 deletions cmd/meowlnir/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ func (m *Meowlnir) Init(ctx context.Context, configPath string, noSaveConfig boo
m.EvaluatorByProtectedRoom = make(map[id.RoomID]*policyeval.PolicyEvaluator)
m.EvaluatorByManagementRoom = make(map[id.RoomID]*policyeval.PolicyEvaluator, len(m.Config.Meowlnir.ManagementRooms))
for _, roomID := range m.Config.Meowlnir.ManagementRooms {
m.EvaluatorByManagementRoom[roomID] = policyeval.NewPolicyEvaluator(m.Client, m.PolicyStore, roomID)
m.EvaluatorByManagementRoom[roomID] = policyeval.NewPolicyEvaluator(m.Client, m.PolicyStore, roomID, m.DB, m.SynapseDB)
}

m.Log.Debug().Msg("Preparing crypto helper")
Expand Down Expand Up @@ -173,7 +173,12 @@ func (m *Meowlnir) ensureBotRegistered(ctx context.Context) {
}

func (m *Meowlnir) Run(ctx context.Context) {
err := m.StateStore.Upgrade(ctx)
err := m.DB.Upgrade(ctx)
if err != nil {
m.Log.WithLevel(zerolog.FatalLevel).Err(err).Msg("Failed to upgrade main db")
os.Exit(20)
}
err = m.StateStore.Upgrade(ctx)
if err != nil {
m.Log.WithLevel(zerolog.FatalLevel).Err(err).Msg("Failed to upgrade state store")
os.Exit(20)
Expand Down
5 changes: 3 additions & 2 deletions config/event.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,9 @@ var (
)

type WatchedPolicyList struct {
RoomID id.RoomID `json:"room_id"`
Name string `json:"name"`
RoomID id.RoomID `json:"room_id"`
Name string `json:"name"`
AutoUnban bool `json:"auto_unban"`
}

type WatchedListsEventContent struct {
Expand Down
65 changes: 64 additions & 1 deletion database/action.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,76 @@
package database

import (
"context"
"time"

"go.mau.fi/util/dbutil"
"maunium.net/go/mautrix/event"
"maunium.net/go/mautrix/id"
)

const (
getTakenActionBaseQuery = `
SELECT target_user, in_room_id, action_type, policy_list, rule_entity, action, taken_at
FROM taken_action
`
getTakenActionsByPolicyListQuery = getTakenActionBaseQuery + `WHERE policy_list=$1`
getTakenActionsByRuleEntityQuery = getTakenActionBaseQuery + `WHERE policy_list=$1 AND rule_entity=$2`
getTakenActionByTargetUserQuery = getTakenActionBaseQuery + `WHERE target_user=$1 AND action_type=$2`
insertTakenActionQuery = `
INSERT INTO taken_action (target_user, in_room_id, action_type, policy_list, rule_entity, action, taken_at)
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (target_user, in_room_id, action_type) DO UPDATE
SET policy_list=excluded.policy_list, rule_entity=excluded.rule_entity, action=excluded.action, taken_at=excluded.taken_at
`
)

type TakenActionQuery struct {
*dbutil.QueryHelper[*TakenAction]
}

func (taq *TakenActionQuery) Put(ctx context.Context, ta *TakenAction) error {
return taq.Exec(ctx, insertTakenActionQuery, ta.sqlVariables()...)
}

func (taq *TakenActionQuery) GetAllByPolicyList(ctx context.Context, policyList id.RoomID) ([]*TakenAction, error) {
return taq.QueryMany(ctx, getTakenActionsByPolicyListQuery, policyList)
}

func (taq *TakenActionQuery) GetAllByRuleEntity(ctx context.Context, policyList id.RoomID, ruleEntity string) ([]*TakenAction, error) {
return taq.QueryMany(ctx, getTakenActionsByRuleEntityQuery, policyList, ruleEntity)
}

func (taq *TakenActionQuery) GetAllByTargetUser(ctx context.Context, userID id.UserID, actionType TakenActionType) ([]*TakenAction, error) {
return taq.QueryMany(ctx, getTakenActionByTargetUserQuery, userID, actionType)
}

type TakenActionType string

const (
TakenActionTypeBanOrUnban TakenActionType = "ban_or_unban"
)

type TakenAction struct {
TargetUser id.UserID
InRoomID id.RoomID
ActionType TakenActionType
PolicyList id.RoomID
RuleEntity string
TargetUser id.UserID
Action event.PolicyRecommendation
TakenAt time.Time
}

func (t *TakenAction) sqlVariables() []any {
return []any{t.TargetUser, t.InRoomID, t.ActionType, t.PolicyList, t.RuleEntity, t.Action, t.TakenAt.UnixMilli()}
}

func (t *TakenAction) Scan(row dbutil.Scannable) (*TakenAction, error) {
var takenAt int64
err := row.Scan(&t.TargetUser, &t.InRoomID, &t.ActionType, &t.PolicyList, &t.RuleEntity, &t.Action, &takenAt)
if err != nil {
return nil, err
}
t.TakenAt = time.UnixMilli(takenAt)
return t, nil
}
9 changes: 9 additions & 0 deletions database/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,23 @@ package database

import (
"go.mau.fi/util/dbutil"

"go.mau.fi/meowlnir/database/upgrades"
)

type Database struct {
*dbutil.Database
TakenAction *TakenActionQuery
}

func New(db *dbutil.Database) *Database {
db.UpgradeTable = upgrades.Table
return &Database{
Database: db,
TakenAction: &TakenActionQuery{
QueryHelper: dbutil.MakeQueryHelper(db, func(qh *dbutil.QueryHelper[*TakenAction]) *TakenAction {
return &TakenAction{}
}),
},
}
}
17 changes: 12 additions & 5 deletions database/upgrades/00-latest.sql
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@
-- v0 -> v1 (compatible with v1+): Latest schema
CREATE TABLE taken_action (
policy_list TEXT NOT NULL,
rule_entity TEXT NOT NULL,
target_user TEXT NOT NULL,
action TEXT NOT NULL,
taken_at BIGINT NOT NULL
target_user TEXT NOT NULL,
in_room_id TEXT NOT NULL,
action_type TEXT NOT NULL,
policy_list TEXT NOT NULL,
rule_entity TEXT NOT NULL,
action TEXT NOT NULL,
taken_at BIGINT NOT NULL,

PRIMARY KEY (target_user, in_room_id, action_type)
);

CREATE INDEX taken_action_list_idx ON taken_action (policy_list);
CREATE INDEX taken_action_entity_idx ON taken_action (policy_list, rule_entity);
17 changes: 7 additions & 10 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,13 @@ go 1.23

require (
github.com/lib/pq v1.10.9
github.com/prometheus/client_golang v1.20.2
github.com/prometheus/client_golang v1.20.3
github.com/rs/zerolog v1.33.0
go.mau.fi/util v0.7.1-0.20240901193650-bf007b10eaf6
go.mau.fi/util v0.7.1-0.20240904173517-ca3b3fe376c2
go.mau.fi/zeroconfig v0.1.3
golang.org/x/exp v0.0.0-20240823005443-9b4947da3948
gopkg.in/yaml.v3 v3.0.1
maunium.net/go/mauflag v1.0.0
maunium.net/go/mautrix v0.20.1-0.20240902204906-db8f2433a1db
maunium.net/go/mautrix v0.20.1-0.20240906145130-6b055b1475bd
)

require (
Expand All @@ -24,7 +23,7 @@ require (
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/mattn/go-sqlite3 v1.14.23 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
Expand All @@ -36,12 +35,10 @@ require (
github.com/tidwall/sjson v1.2.5 // indirect
github.com/yuin/goldmark v1.7.4 // indirect
golang.org/x/crypto v0.26.0 // indirect
golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/sys v0.24.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/text v0.18.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
)

//replace maunium.net/go/mautrix => ../mautrix-go
//replace go.mau.fi/util => ../../Go/go-util
24 changes: 12 additions & 12 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -31,17 +31,17 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0=
github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw=
github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg=
github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4=
github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
Expand All @@ -66,8 +66,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
go.mau.fi/util v0.7.1-0.20240901193650-bf007b10eaf6 h1:cSLCabMKbR6rTPYRGWD2XaHo210BK3BtPg+CRC4A4og=
go.mau.fi/util v0.7.1-0.20240901193650-bf007b10eaf6/go.mod h1:WuAOOV0O/otkxGkFUvfv/XE2ztegaoyM15ovS6SYbf4=
go.mau.fi/util v0.7.1-0.20240904173517-ca3b3fe376c2 h1:VZQlKBbeJ7KOlYSh6BnN5uWQTY/ypn/bJv0YyEd+pXc=
go.mau.fi/util v0.7.1-0.20240904173517-ca3b3fe376c2/go.mod h1:WgYvbt9rVmoFeajP97NunQU7AjgvTPiNExN3oTHeePs=
go.mau.fi/zeroconfig v0.1.3 h1:As9wYDKmktjmNZW5i1vn8zvJlmGKHeVxHVIBMXsm4kM=
go.mau.fi/zeroconfig v0.1.3/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70=
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
Expand All @@ -79,10 +79,10 @@ golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand All @@ -94,5 +94,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M=
maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA=
maunium.net/go/mautrix v0.20.1-0.20240902204906-db8f2433a1db h1:kZ4iZLIPRZD62wqcnGwRATJCNjU6EC8vfEOewwX9pZQ=
maunium.net/go/mautrix v0.20.1-0.20240902204906-db8f2433a1db/go.mod h1:IXDDoX+dqBkNnrjDMouE3FUExiR+hhmaEFsvXG3HzfQ=
maunium.net/go/mautrix v0.20.1-0.20240906145130-6b055b1475bd h1:gfiJD2cPS9iUek1UI+DOUn08zogF4kmu7XYfBqSrAU4=
maunium.net/go/mautrix v0.20.1-0.20240906145130-6b055b1475bd/go.mod h1:l6nYvD5/FMSrAZ/IP1AqJV0b47SRl/0uQNRiy4CcSVk=
67 changes: 60 additions & 7 deletions policyeval/evaluate.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ import (
"slices"

"github.com/rs/zerolog"
"maunium.net/go/mautrix/event"
"maunium.net/go/mautrix/id"

"go.mau.fi/meowlnir/database"
"go.mau.fi/meowlnir/policylist"
)

Expand All @@ -20,26 +22,77 @@ func (pe *PolicyEvaluator) EvaluateAll(ctx context.Context) {

func (pe *PolicyEvaluator) EvaluateAllMembers(ctx context.Context, members []id.UserID) {
for _, member := range members {
pe.EvaluateNewMember(ctx, member)
pe.EvaluateUser(ctx, member)
}
}

func (pe *PolicyEvaluator) EvaluateNewMember(ctx context.Context, userID id.UserID) {
func (pe *PolicyEvaluator) EvaluateUser(ctx context.Context, userID id.UserID) {
match := pe.Store.MatchUser(pe.GetWatchedLists(), userID)
if match == nil {
return
}
zerolog.Ctx(ctx).Info().
zerolog.Ctx(ctx).Debug().
Stringer("user_id", userID).
Any("recommendation", match.Recommendations()).
Any("matches", match).
Msg("Matched user in membership event")
Msg("Found matches for user")
pe.ApplyPolicy(ctx, userID, match)
}

func (pe *PolicyEvaluator) EvaluateRemovedRule(ctx context.Context, policy *policylist.Policy) {
// TODO
if policy.Recommendation == event.PolicyRecommendationUnban {
// When an unban rule is removed, evaluate all joined users against the removed rule
// to see if they should be re-evaluated against all rules (and possibly banned)
pe.usersLock.RLock()
users := slices.Collect(maps.Keys(pe.users))
pe.usersLock.RUnlock()
for _, userID := range users {
if policy.Pattern.Match(string(userID)) {
pe.EvaluateUser(ctx, userID)
}
}
} else {
// For ban rules, find users who were banned by the rule and re-evaluate them.
reevalTargets, err := pe.DB.TakenAction.GetAllByRuleEntity(ctx, policy.RoomID, policy.Entity)
if err != nil {
zerolog.Ctx(ctx).Err(err).Str("policy_entity", policy.Entity).
Msg("Failed to get actions taken for removed policy")
pe.sendNotice(ctx, "Database error in EvaluateRemovedRule (GetAllByRuleEntity): %v", err)
return
}
pe.ReevaluateActions(ctx, reevalTargets)
}
}

func (pe *PolicyEvaluator) EvaluateAddedRule(ctx context.Context, policy *policylist.Policy) {
// TODO
pe.usersLock.RLock()
users := slices.Collect(maps.Keys(pe.users))
pe.usersLock.RUnlock()
for _, userID := range users {
if policy.Pattern.Match(string(userID)) {
pe.ApplyPolicy(ctx, userID, policylist.Match{policy})
}
}
}

func (pe *PolicyEvaluator) ReevaluateAffectedByLists(ctx context.Context, policyLists []id.RoomID) {
var reevalTargets []*database.TakenAction
for _, list := range policyLists {
targets, err := pe.DB.TakenAction.GetAllByPolicyList(ctx, list)
if err != nil {
zerolog.Ctx(ctx).Err(err).Stringer("policy_list_id", list).
Msg("Failed to get actions taken from policy list")
pe.sendNotice(ctx, "Database error in ReevaluateAffectedByLists (GetAllByPolicyList): %v", err)
continue
}
if reevalTargets == nil {
reevalTargets = targets
} else {
reevalTargets = append(reevalTargets, targets...)
}
}
pe.ReevaluateActions(ctx, reevalTargets)
}

func (pe *PolicyEvaluator) ReevaluateActions(ctx context.Context, actions []*database.TakenAction) {

}
Loading

0 comments on commit 4e6fd57

Please sign in to comment.