diff --git a/channel/github-discussions/index.html b/channel/github-discussions/index.html deleted file mode 100644 index 83e1886..0000000 --- a/channel/github-discussions/index.html +++ /dev/null @@ -1,6687 +0,0 @@ - - - - - - Slack Export - #github-discussions - - - - - -
- - - -
- - - -
-
- - - - -
- -
Julien Le Dem - (julien@apache.org) -
-
2023-02-03 17:55:40
 -
-

@Julien Le Dem has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Julien Le Dem - (julien@apache.org) -
-
2023-02-03 17:56:32
 -
-

/github subscribe OpenLineage/OpenLineage discussions

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Julien Le Dem - (julien@apache.org) -
-
2023-02-03 17:57:39
 -
-

/github subscribe OpenLineage/OpenLineage discussions

- - - -
-
-
-
- - - - - -
-
- -
- - -
- -
B01UH3Z8K8V - -
-
2023-02-03 17:57:39
 -
-

✅ Subscribed to OpenLineage/OpenLineage. This channel will receive notifications for issues, pulls, commits, releases, deployments, discussions

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Michael Robinson - (michael.robinson@astronomer.io) -
-
2023-02-03 21:22:08
 -
-

@Michael Robinson has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Mike Dillion - (mike.dillion@gmail.com) -
-
2023-02-11 18:51:36
 -
-

@Mike Dillion has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
jrich - (jasonrich85@icloud.com) -
-
2023-03-10 14:52:18
 -
-

@jrich has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Dev Jadhav - (dev.jadhav@loxsolution.com) -
-
2023-04-07 08:31:49
 -
-

@Dev Jadhav has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Sudhar Balaji - (sudharshan.dataaces@gmail.com) -
-
2023-04-25 07:04:19
 -
-

@Sudhar Balaji has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Yuanli Wang - (yuanliw@bu.edu) -
-
2023-05-25 20:33:51
 -
-

@Yuanli Wang has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Nam Nguyen - (nam@astrafy.io) -
-
2023-07-14 05:37:30
 -
-

@Nam Nguyen has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Glyn Bowden (HPE) - (glyn.bowden@hpe.com) -
-
2023-08-07 13:49:30
 -
-

@Glyn Bowden (HPE) has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
GTC - (tsungchih.hd@gmail.com) -
-
2023-10-21 04:59:15
 -
-

@GTC has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Sheeri Cabral (Collibra) - (sheeri.cabral@collibra.com) -
-
2023-11-09 15:30:23
 -
-

@Sheeri Cabral (Collibra) has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Sheeri Cabral (Collibra) - (sheeri.cabral@collibra.com) -
-
2023-11-09 15:30:58
 -
-

Let me know if I did that wrong ^^^ It’s been a while since I modified someone else’s PR with my own commits.

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Harel Shein - (harel.shein@gmail.com) -
-
2023-11-16 11:40:06
 -
-

@Harel Shein has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Gowthaman Chinnathambi - (gowthamancdev@gmail.com) -
-
2024-01-18 23:26:56
 -
-

@Gowthaman Chinnathambi has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
jayant joshi - (itsjayantjoshi@gmail.com) -
-
2024-01-24 01:10:27
 -
-

@jayant joshi has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
tati - (tatiana.alchueyr@astronomer.io) -
-
2024-01-30 12:12:58
 -
-

@tati has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Ewan Lord - (ewanlord@gmail.com) -
-
2024-01-31 05:28:13
 -
-

@Ewan Lord has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Josh Fischer - (josh@joshfischer.io) -
-
2024-02-03 20:13:56
 -
-

@Josh Fischer has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
assia fellague - (assia.fellague@canal-plus.com) -
-
2024-02-22 11:15:06
 -
-

@assia fellague has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Santiago Cobos - (santiago.cobos@ibm.com) -
-
2024-03-25 16:42:24
 -
-

@Santiago Cobos has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Ray Lacerda - (ray.lacerda@live.com) -
-
2024-03-27 21:42:13
 -
-

@Ray Lacerda has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Ray Lacerda - (ray.lacerda@live.com) -
-
2024-03-27 21:42:20
 -
-

@Ray Lacerda has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - \ No newline at end of file diff --git a/channel/github-notifications/index.html b/channel/github-notifications/index.html deleted file mode 100644 index e9331b5..0000000 --- a/channel/github-notifications/index.html +++ /dev/null @@ -1,40921 +0,0 @@ - - - - - - Slack Export - #github-notifications - - - - - -
- - - -
- - - -
-
- - - - -
- -
Julien Le Dem - (julien@apache.org) -
-
2020-11-03 16:50:07
 -
-

@Julien Le Dem has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Jørn Hansen - (jornhansen@gmail.com) -
-
2020-12-19 06:25:50
 -
-

@Jørn Hansen has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Ananth Packkildurai - (vananth22@gmail.com) -
-
2020-12-19 15:05:57
 -
-

@Ananth Packkildurai has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Harikiran Nayak - (hari@streamsets.com) -
-
2020-12-21 15:12:21
 -
-

@Harikiran Nayak has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Willy Lulciuc - (willy@datakin.com) -
-
2020-12-22 19:55:31
 -
-

@Willy Lulciuc has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Alagappan Sethuraman - (alagappan.als@gmail.com) -
-
2020-12-23 15:31:21
 -
-

@Alagappan Sethuraman has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Laurent Paris - (laurent@datakin.com) -
-
2021-02-01 17:51:37
 -
-

@Laurent Paris has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
dorzey - (dorzey@gmail.com) -
-
2021-02-02 07:05:40
 -
-

@dorzey has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Alexander Gilfillan - (agilfillan@dealerinspire.com) -
-
2021-02-02 19:19:25
 -
-

@Alexander Gilfillan has joined the channel

- - - -
-
-
-
- - - - - - - - - -
-
- - - - -
- -
Girish Lingappa - (glingappa@netflix.com) -
-
2021-02-04 14:56:07
 -
-

@Girish Lingappa has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Edgar Ramírez Mondragón - (edgarrm358@gmail.com) -
-
2021-02-08 03:26:38
 -
-

@Edgar Ramírez Mondragón has joined the channel

- - - -
-
-
-
- - - - - - - - - -
-
- - - - -
- -
Xinbin Huang - (bin.huangxb@gmail.com) -
-
2021-02-11 14:09:49
 -
-

@Xinbin Huang has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
aliou - (aliouamardev@gmail.com) -
-
2021-02-16 11:01:31
 -
-

@aliou has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Arthur Wiedmer - (awiedmer@apple.com) -
-
2021-02-25 17:43:24
 -
-

@Arthur Wiedmer has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Victor Shafran - (victor.shafran@databand.ai) -
-
2021-03-09 16:44:03
 -
-

@Victor Shafran has joined the channel

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Michael Collado - (collado.mike@gmail.com) -
-
2021-04-02 11:59:30
 -
-

@Michael Collado has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Ross Turk - (ross@datakin.com) -
-
2021-04-03 21:15:06
 -
-

@Ross Turk has joined the channel

- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub (Legacy) - -
-
2021-04-16 13:41:47
 -
-

GitHub app is successfully upgraded in your workspace 🎉 -To receive notifications in your private channels, you need to invite the GitHub app /invite @GitHub

- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-04-16 14:49:42
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/36|#36 Adding Marquez implementation links>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-04-16 14:51:06
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/32|#32 Add Egeria to related projects>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-04-16 17:59:27
 -
-

[OpenLineage/OpenLineage] Pull request opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/43|#43 add Governance>
-
- <p>I worked with <a href="https://github.com/mandy-chessell">@mandy-chessell</a> to prepare a first version of the OpenLineage governance.</p> -
- -
-
Reviewers
- rdblue, jcampbell, drewbanin, wslulciuc, mobuchowski, henneberger, mandy-chessell, collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-04-22 19:25:58
 -
-

[OpenLineage/OpenLineage] Pull request opened by MansurAshraf

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/44|#44 Fix Egeria's url in readme>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-04-23 09:54:40
 -
-

[OpenLineage/OpenLineage] Issue opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/45|#45 [PROPOSAL] Standarize custom facet naming>
-
- <p><strong>Purpose:</strong><br /> -Main motivation of this proposal is making sure that custom facets in different implementations of OpenLineage clients for the same data source have the same name. Another advantage will be decreased chance of different facets name colliding.</p> - -<p><strong>Proposed implementation</strong><br /> -I propose that custom facets should follow name pattern <code>{prefix}{name}{entity}Facet</code> PascalCased, where</p> - -<p>• Prefix must be distinct identifier named after the project -• Name is the desired facet name -• Entity is the core OpenLineage entity for which the facet will be attached</p> - -<p>Important aspect for OpenLineage consumers is standarizing JSON key naming.<br /> -I propose to follow pattern <code>{prefix}_{name}</code> snakeCased.</p> - -<p>Example of custom facet following both patterns: <code>BigQueryStatisticsJobFacet</code> and <code>bigQuery_statistics</code></p> -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-04-23 09:55:12
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/46|#46 Standarize custom facet naming>
-
- <p>Proposed spec change of <a href="https://github.com/OpenLineage/OpenLineage/issues/45">#45</a></p> - -<p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-04-26 21:21:56
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/44|#44 Fix Egeria's url in readme>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-04-28 17:12:39
 -
-

[OpenLineage/OpenLineage] Pull request opened by jquintus

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/47|#47 Fix a few minor typos>
-
- <p>These changes fix typos in the documentation like</p> - -<p>• Changing an instance of "and" to "an" -• Spelling</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-06 11:59:11
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/43|#43 add Governance>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-06 12:00:00
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/46|#46 Standarize custom facet naming>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-06 12:02:25
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/47|#47 Fix a few minor typos>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-06 12:16:58
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/48|#48 ci: release client to pypi>
-
- <p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-06 20:42:26
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/31|#31 Create unique naming spec>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Harshal Sheth - (harshal@acryl.io) -
-
2021-05-12 17:48:13
 -
-

@Harshal Sheth has joined the channel

- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
Ross Turk - (ross@datakin.com) -
-
2021-05-14 14:23:41
 -
-

*Thread Reply:* shh don't tell anyone there's a website coming

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-17 08:40:15
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/49|#49 default schema field description should not serialize to empty string>
-
- <p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-17 20:52:01
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/49|#49 default schema field description should not serialize to empty string>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-17 20:52:18
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/48|#48 ci: release client to pypi>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-18 08:46:42
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/45|#45 [PROPOSAL] Standarize custom facet naming>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-18 08:47:39
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/33|#33 [PROPOSAL] Add standard python client>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-20 09:38:09
 -
-

[OpenLineage/OpenLineage] New release Release - published by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/releases/tag/0.0.1-rc.1|Release - >
-
- <p>Testing release of Python OpenLineage client.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-20 11:25:56
 -
-

[OpenLineage/OpenLineage] Pull request closed by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/30|#30 OpenLineage LFAI&amp;Data proposal>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-20 15:27:30
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/50|#50 Add license header to source files>
-
- <p><strong>SPDX-License-Identifier: Apache-2.0</strong></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-20 20:33:16
 -
-

[OpenLineage/OpenLineage] Pull request opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/51|#51 Add best practices badge to README.md>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - -
- - - } - - DCO: DCO - (https://github.com/OpenLineage/OpenLineage/runs/2635194679) -
- - - -
-
- -
- - - - - - - - -
-
- - - - -
-
- ✅ No checks have passed -
- - - - - - - -
- - 0/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-21 07:56:39
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/52|#52 ci/python: format RC versions compliant to PEP 440, use tag to version if exists, unpin dependencies>
-
- <p><code>0.0.1-rc.1</code> -&gt; <code>0.0.1rc1</code></p> - -<p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- -
-
Reviewers
- julienledem, wslulciuc, collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-21 10:49:46
 -
-

[OpenLineage/OpenLineage] Pull request merged by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/52|#52 ci/python: format RC versions compliant to PEP 440, use tag to version if exists, unpin dependencies>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-21 10:59:23
 -
-

[OpenLineage/OpenLineage] New release Release - 0.0.1-rc2 published by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/releases/tag/0.0.1-rc2|Release - 0.0.1-rc2>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-21 11:04:10
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/53|#53 ci/python: fix wheel version name>
-
- <p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- -
-
Comments
- 1 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-21 11:05:59
 -
-

[OpenLineage/OpenLineage] Pull request merged by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/53|#53 ci/python: fix wheel version name>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-21 11:07:05
 -
-

[OpenLineage/OpenLineage] New release Release - 0.0.1-rc3 published by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/releases/tag/0.0.1-rc3|Release - 0.0.1-rc3>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-21 12:33:47
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/39|#39 Release OpenLineage python client to pipy>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-25 12:02:02
 -
-

[OpenLineage/OpenLineage] Issue opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/55|#55 Java client should support Java 8>
-
- -
- -
-
Assignees
- mobuchowski -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-26 07:47:32
 -
-

[OpenLineage/OpenLineage] Pull request closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/54|#54 ci: publish java client>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-26 07:47:57
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/56|#56 ci: publish java client>
-
- <p>Also, make it compile on Java 8</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-26 11:51:30
 -
-

[OpenLineage/OpenLineage] New release Release - 0.0.0-rc4 published by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/releases/tag/0.0.0-rc4|Release - 0.0.0-rc4>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-27 09:53:39
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/38|#38 prototype data metrics facet with GreatExpectations>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-28 04:30:21
 -
-

[OpenLineage/OpenLineage] Pull request merged by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/56|#56 ci: publish java client>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - - - - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-28 20:08:27
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/57|#57 Import marquez-spark integration to OpenLineage integrations>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-05-28 20:08:33
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/58|#58 Import Marquez-airflow integration to OpenLineage integrations>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-01 11:10:54
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/55|#55 Java client should support Java 8>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-02 10:20:27
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/59|#59 python: serialization: fixed handling of nested nulls in dictionaries>
-
- -
- -
-
Reviewers
- julienledem, wslulciuc, collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-04 22:05:33
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/51|#51 Add best practices badge to README.md>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-08 17:26:30
 -
-

[OpenLineage/website] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/website/pull/1|#1 Add OL getting started page>
-
- <p>This PR adds the <em>Getting Started</em> page for OpenLineage using Marquez as the HTTP backend to send run-level events.</p> - -<p><a href="https://github.com/rossturk">@rossturk</a>: Few things I might need help with:</p> - -<p>• I couldn't get the Table of Contents (top of page) to link to sections in the page (or we can leave the format as is) -• The code blocks with the <code>curl</code> cmds aren't really formatted that well. Did you have a plugin in mind?</p> -
- -
-
Reviewers
- rossturk -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-09 02:54:03
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/59|#59 python: serialization: fixed handling of nested nulls in dictionaries>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-09 16:14:28
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/1|#1 Add OL getting started page>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-09 18:00:27
 -
-

[OpenLineage/OpenLineage] New release Release - 0.0.1-rc5 published by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/releases/tag/0.0.1-rc5|Release - 0.0.1-rc5>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-10 07:40:24
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/60|#60 python: relax client requirements>
-
- <p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-10 07:53:45
 -
-

[OpenLineage/OpenLineage] New release Release - 0.0.1-rc6 published by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/releases/tag/0.0.1-rc6|Release - 0.0.1-rc6>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-10 16:47:31
 -
-

[OpenLineage/website] Pull request opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/website/pull/2|#2 Add content for spark lineage collection>
-
- <p>I did my best re: the syntax highlighting and the code box scrolling, but here I think it's best to give it someone who knows this stuff better than I do :)</p> -
- -
-
Reviewers
- rossturk -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-10 21:32:40
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/2|#2 Add content for spark lineage collection>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-11 12:02:28
 -
-

[OpenLineage/website] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/website/pull/3|#3 Add OL integration with Airflow page>
-
- <p>This PR adds the OpenLineage integration page for Airflow.</p> -
- -
-
Reviewers
- rossturk -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-14 12:33:26
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/3|#3 Add OL integration with Airflow page>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-16 10:59:14
 -
-

[OpenLineage/OpenLineage] Issue opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/61|#61 Releasing via CI>
-
- <p>With the upcoming release of OpenLineage <code>0.1.0</code>, we'll want to publish our artifacts via CI. Below, I propose the following release steps:</p> - -<ol><li>Bump the version in <code>gradle.properties</code> for <code>openlineage-java</code> and in <code>setup.py</code> for <code>openlineage-python</code></li><li>Tag the release with <code>X.Y.Z</code> on <code>main</code> branch</li><li>Prepare the next development version</li><li>CI kickoffs release jobs to publish artifacts</li> -</ol> - -<p>&gt; <strong>Note:</strong> The steps outlined above are inspired by the release steps for <a href="https://github.com/MarquezProject/marquez/blob/main/RELEASING.md"><code>Marquez</code></a>. Also, for <strong>step 1</strong>, we can reference <a href="https://github.com/MarquezProject/marquez/blob/main/new-version.sh"><code>new-version.sh</code></a> used on Marquez releases.</p> - -<p><a href="https://github.com/mobuchowski">@mobuchowski</a>: You've already made some great additions to our CI config, this is just to capture the CI release flow.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-16 11:05:51
 -
-

[OpenLineage/OpenLineage] Issue opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/62|#62 OpenLineage artifacts published on PRs>
-
- <p>OpenLineage artifacts (<code>openlineage-java</code>, <code>openlineage-python</code>) are currently being published in PRs, see <a href="https://github.com/OpenLineage/OpenLineage/pull/60">#60</a>. We'll only want to publish artifacts on the <code>main</code> branch.</p> - -<p>Related to <a href="https://github.com/OpenLineage/OpenLineage/issues/61">#61</a></p> -
- -
-
Labels
- bug -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-16 18:49:51
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/63|#63 [PROPOSAL] Define the OpenLineage spec versioning mechanism>
-
- <p><strong>Purpose:</strong><br /> -We need to define how the OpenLineage spec gets versioned and published.<br /> -Some requirements:</p> - -<p>• The OpenLineage spec and related libraries are in the OpenLineage repo. -• The OpenLineage spec version should change only when the spec itself changes. -• The libraries in the repo change more frequently than the spec (including when the spec changes). -• We want to version the OpenLineage spec independently of the api spec. -• The mechanism to version and publish the OpenLineage core spec, should apply to publishing custom facets.</p> - -<p><strong>Proposed implementation</strong><br /> -I have started a google doc to make it easier to collaborate:<br /> -<a href="https://docs.google.com/document/d/1inhmb90SB62VyYf8nkkpjBDPpyr2wxkIXQ_AFIQiF9I/edit">https://docs.google.com/document/d/1inhmb90SB62VyYf8nkkpjBDPpyr2wxkIXQ_AFIQiF9I/edit</a><br /> -please comment in the doc.</p> -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-16 19:00:20
 -
-

[OpenLineage/OpenLineage.github.io] Pull request opened by rossturk

-
- - - - -
<https://github.com/OpenLineage/OpenLineage.github.io/pull/1|#1 Initial import of openlineage web content>
-
- <p>Signed-off-by: Ross Turk <a href="mailto:ross@rossturk.com">ross@rossturk.com</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage.github.io">OpenLineage/OpenLineage.github.io</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-16 19:03:38
 -
-

[OpenLineage/OpenLineage.github.io] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage.github.io/pull/1|#1 Initial import of openlineage web content>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage.github.io">OpenLineage/OpenLineage.github.io</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-16 19:40:28
 -
-

[OpenLineage/website] is now public!

-
- - -
- - - } - - OpenLineage - (https://github.com/OpenLineage) -
- - - -
-
- -
- - - - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
- ❤️ Willy Lulciuc, Ross Turk -
- -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-16 19:58:29
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/64|#64 Automate publishing of gatsby website to github pages.>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-16 20:26:43
 -
-

[OpenLineage/website] Pull request opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/website/pull/4|#4 adding blog post about parallels of OpenLineage and OpenTelemetry>
-
- <p>Signed-off-by: Julien Le Dem <a href="mailto:julien@apache.org">julien@apache.org</a></p> -
- -
-
Reviewers
- rossturk -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - -
- - - } - - DCO: DCO - (https://github.com/OpenLineage/website/runs/2844505752) -
- - - -
-
- -
- - - - - - - - -
-
- - - - -
-
- ✅ No checks have passed -
- - - - - - - -
- - 0/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-16 22:07:41
 -
-

[OpenLineage/website] Pull request opened by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/5|#5 Quick fixes>
-
- <p>A series of quick fixes to our new website:</p> - -<p>• hid cookie banner that linked to a nonexistent cookie policy -• google analytics tracking -• added author field to blogs -• added a favicon</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-17 04:18:34
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/65|#65 Set `Run.runId` format as uuid, according to RFC4122>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-17 05:50:20
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/62|#62 OpenLineage artifacts published on PRs>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-17 13:56:41
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/5|#5 Quick fixes>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-17 13:59:09
 -
-

[OpenLineage/website] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/website/pull/6|#6 Update tip in getting started page to use --tag X.Y.Z>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-17 14:26:56
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/6|#6 Update tip in getting started page to use --tag X.Y.Z>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-17 14:28:14
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/4|#4 adding blog post about parallels of OpenLineage and OpenTelemetry>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-17 16:22:44
 -
-

[OpenLineage/website] Pull request opened by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/7|#7 Updates to parallels blog post + updated front page doc links>
-
- <p>Signed-off-by: Ross Turk <a href="mailto:ross@rossturk.com">ross@rossturk.com</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-17 18:07:08
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/7|#7 Updates to parallels blog post + updated front page doc links>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-17 18:20:34
 -
-

[OpenLineage/OpenLineage.github.io] Pull request opened by rossturk

-
- - - - -
<https://github.com/OpenLineage/OpenLineage.github.io/pull/2|#2 New blog post + series of small refinements>
-
- <p>Signed-off-by: Ross Turk <a href="mailto:ross@rossturk.com">ross@rossturk.com</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage.github.io">OpenLineage/OpenLineage.github.io</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-17 18:53:00
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/66|#66 Clarify the job naming strategy>
-
- <p>We need a spec similar to the dataset naming strategy for jobs</p> -
- -
-
Assignees
- julienledem -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-17 21:16:47
 -
-

[OpenLineage/OpenLineage.github.io] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage.github.io/pull/2|#2 New blog post + series of small refinements>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage.github.io">OpenLineage/OpenLineage.github.io</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-18 15:16:09
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/67|#67 Add a Consumer spec>
-
- <p>Consumers should be able to receive any facets and need a JSONSCHEMA that just defines the core model without any facet</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-18 15:18:03
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/68|#68 Finalize the client APIs spec>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-18 17:12:11
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/65|#65 Set `Run.runId` format as uuid, according to RFC4122>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-21 09:39:30
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/69|#69 python client: enforce runId being UUID>
-
- <p>Python equivalent of Java PR:<br /> -<a href="https://github.com/OpenLineage/OpenLineage/pull/65">#65</a></p> -
- -
-
Reviewers
- julienledem -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-21 17:28:08
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/69|#69 python client: enforce runId being UUID>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-22 05:48:49
 -
-

[OpenLineage/OpenLineage] Pull request merged by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/60|#60 python: relax client requirements>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-25 15:36:47
 -
-

[OpenLineage/website] Pull request opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/website/pull/8|#8 Added lineage API exploration blog>
-
- <p>Notes</p> - -<p>• The post is ~1800 words excluding bash commands and JSON output -• I copied over the banner and other images from another blog folder -• I put placeholders in the title and description fields, but they probably could use some attention. -• The section headers might need some tweaking- I tried to add markers for each data model I discussed as well as a section header for each goal being tackled -• It's easier to read from the running instance, rather than reading the mdx directly- just because all the json and bash aren't as clearly dileneated</p> -
- -
-
Reviewers
- rossturk -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-25 16:32:06
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/70|#70 Add a Kafka backend for sending lineage events>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-25 17:46:16
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/71|#71 Finalize the producer-oriented Openlineage spec for 0.1>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-25 19:02:10
 -
-

[OpenLineage/website] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/website/pull/9|#9 Draft of backfilling airflow DAGs using marquez blog>
-
- <p>This PR adds the a draft version of the <em>Backfilling Airflow DAGs using Marquez</em> blog.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-28 14:44:38
 -
-

[OpenLineage/website] Issue opened by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/issues/10|#10 Code blocks contain extraneous new line at bottom>
-
- <p>For some reason, when Prism deals with the syntax highlighting it adds an extra empty line to the bottom of each code block.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-29 05:22:07
 -
-

[OpenLineage/OpenLineage] Issue opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/72|#72 Drop env var `OPENLINEAGE_PRODUCER` usage in py client>
-
- <p>The <code>py</code> client currently allows for setting the openlineage producer via an env var: <code>OPENLINEAGE_PRODUCER</code>. We'll want allow users to configure the producer via the client constructor.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-06-29 20:26:18
 -
-

[OpenLineage/OpenLineage] Issue opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/73|#73 Move Marquez integrations to OpenLineage>
-
- <p>The <a href="https://github.com/MarquezProject/marquez">Marquez</a> community is excited to announce that the following integrations will be moved to the wider OpenLineage ecosystem:</p> - -<p>• <a href="https://github.com/MarquezProject/marquez/tree/main/integrations/airflow"><code>marquez-airflow</code></a>: A library that integrates Airflow DAGs to emit OpenLineage metadata. -• <a href="https://github.com/MarquezProject/marquez/tree/main/integrations/spark"><code>marquez-spark</code></a>: A spark agent using jvm instrumentation to emit OpenLineage metadata. -• <a href="https://github.com/MarquezProject/marquez/tree/main/integrations/common"><code>marquez-integration-common</code></a>: Shared code across integrations. -• <code>marquez-dbt-snowflake</code>: A library that integrates with DBT to emit OpenLineage metadata for Snowflake. -• <code>marquez-dbt-bigquery</code>: A library that integrates with DBT to emit OpenLineage metadata for BigQuery.</p> - -<p>&gt; Note: You can view the integrations in the Marquez repo under <a href="https://github.com/MarquezProject/marquez/tree/main/integrations"><code>marquez/integrations/</code></a>. Marquez is an official reference implementation of the OpenLineage API and all the integrations adopt the OpenLineage standard.</p> - -<p><strong>Why are the integrations being moved to OpenLineage?</strong></p> - -<p>The OpenLineage standard was influenced by Marquez's underlying data model (<em>datasets</em>, <em>jobs</em>, <em>runs</em>) while also drawing inspiration from the projects lineage ingestion API. With OpenLineage now a standalone project under <a href="https://lfaidata.foundation">LFAI &amp; Data Foundation</a>, we've decided that now would be the right time to move the integrations over as more and more integrations are being added to the <a href="https://github.com/OpenLineage/OpenLineage/projects/1">projects roadmap</a> (<em>Looker</em>, <em>Flink</em>, <em>Beam</em>, etc). This would also allow for the OpenLineage community to grow and learn from existing integration implementations, prioritize bug fixes, propose new integrations and changes to the spec, etc.</p> - -<p><strong>What is the timeline for moving the integrations to OpenLineage?</strong></p> - -<p>Below we outline a high-level timeline of moving the integrations to OpenLineage. We encourage everyone to provide feedback and express any concerns you may have by commenting in this issue:</p> - -<p>☐ Release Marquez <a href="https://github.com/MarquezProject/marquez/projects/12"><code>0.16.0</code></a> -☐ Move the following integrations from Marquez to OpenLineage: - ☐ <code>marquez-airflow</code> - ☐ <code>marquez-spark</code> - ☐ <code>marquez-integration-common</code> - ☐ <code>marquez-dbt-snowflake</code> - ☐ <code>marquez-dbt-bigquery</code> -☐ Deprecate all integrations in Marquez, then link to the new location in the OpenLineage repo</p> - -<p>Then, going forward, new integrations and improvements will be made by open pull requests against the OpenLineage repo. <strong>Note that during this migration phase, existing Marquez specific functionality and environment variables will be maintained to ensure backwards compatibility. The main difference is that integration packages will get a new name: <code>openlineage-**</code> instead of <code>marquez-**</code></strong>.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-01 12:21:57
 -
-

[OpenLineage/OpenLineage.github.io] Pull request opened by rossturk

-
- - - - -
<https://github.com/OpenLineage/OpenLineage.github.io/pull/3|#3 New build with the backfill blog>
-
- <p>Signed-off-by: Ross Turk <a href="mailto:ross@datakin.com">ross@datakin.com</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage.github.io">OpenLineage/OpenLineage.github.io</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-01 14:25:45
 -
-

[OpenLineage/website] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/website/pull/9|#9 Backfilling Airflow DAGs using Marquez>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-01 14:37:04
 -
-

[OpenLineage/website] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/website/pull/11|#11 Update code styling for backfill blog>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-01 14:38:23
 -
-

[OpenLineage/website] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/website/pull/11|#11 Update code styling for backfill blog>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-01 14:44:34
 -
-

[OpenLineage/website] Issue opened by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/issues/12|#12 Blogs on index page are not listed in reverse chrono order>
-
- <p>I suspect the graphql query on the index page.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 10:54:17
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/74|#74 Move integration/common from marquez>
-
- <p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- -
-
Comments
- 1 -
- -
-
Reviewers
- wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 16:49:09
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/75|#75 Add documentation on how to create custom facets>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 16:49:13
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/76|#76 Improve contribution guidelines. How to create a new Facet>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 16:50:10
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/77|#77 Add support for Looker lineage>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 16:50:14
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/78|#78 Add support for tableau lineage>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 16:50:19
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/79|#79 Add Flink integration>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 16:50:24
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/80|#80 Add Beam integration>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 16:50:47
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/81|#81 Add Prefect support>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 16:51:05
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/82|#82 Add dbt support>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 16:59:18
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/83|#83 Finalize the client API spec>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 16:59:48
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/84|#84 Finalize the OpenLineage mission statement>
-
- <p>It is part of our technical charter.<br /> -Currently we have: "to enable the industry at-large to collect lineage metadata consistently and comprehensively across complex pipelines, creating a deeper understanding of data."</p> - -<p>We'd want to capture that this is specifically operational lineage, or lineage and metadata collected as the job is running.</p> - -<p>It is also something more widely applicable than just data pipelines, and should reflect that,</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 17:14:07
 -
-

[OpenLineage/OpenLineage.github.io] Pull request closed by rossturk

-
- - - - -
<https://github.com/OpenLineage/OpenLineage.github.io/pull/3|#3 New build with the backfill blog>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage.github.io">OpenLineage/OpenLineage.github.io</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-02 17:24:26
 -
-

[OpenLineage/website] Pull request opened by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/13|#13 Order the homepage blogs by date in descending order>
-
- <p>Signed-off-by: Ross Turk <a href="mailto:ross@rossturk.com">ross@rossturk.com</a></p> - -<p>Thanks <a href="https://github.com/phixMe">@phixMe</a> and <a href="https://github.com/wslulciuc">@wslulciuc</a> for spotting this and suggesting the solution!</p> -
- -
-
Reviewers
- phixMe -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-07 19:59:01
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/86|#86 Add backend for sending lineage to Amundsen>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-07 22:27:50
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/8|#8 Added lineage API exploration blog>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-07 22:28:09
 -
-

[OpenLineage/website] Issue closed by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/issues/12|#12 Blogs on index page are not listed in reverse chrono order>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-07 22:28:11
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/13|#13 Order the homepage blogs by date in descending order>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-07 22:36:58
 -
-

[OpenLineage/website] Pull request opened by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/14|#14 Update date and summary for the lineage API blog>
-
- -
- -
-
Reviewers
- collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - -
- - - } - - DCO: DCO - (https://github.com/OpenLineage/website/runs/3015059078) -
- - - -
-
- -
- - - - - - - - -
-
- - - - -
-
- ✅ No checks have passed -
- - - - - - - -
- - 0/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-08 03:13:17
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/14|#14 Update date and summary for the lineage API blog>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-08 20:01:04
 -
-

[OpenLineage/website] Pull request opened by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/15|#15 New feature/banner images for lineage post>
-
- <p>Signed-off-by: Ross Turk <a href="mailto:ross@datakin.com">ross@datakin.com</a></p> - -<p>Here are some screenshots of the artwork. Let me know if you would like me to try something different.</p> - -<p><a href="https://user-images.githubusercontent.com/1434475/125004197-03618080-e027-11eb-954a-45dee8ea14e0.png">Screen Shot 2021-07-08 at 7 58 21 PM</a></p> - -<p><a href="https://user-images.githubusercontent.com/1434475/125004200-0492ad80-e027-11eb-986f-56e8c72d93af.png">Screen Shot 2021-07-08 at 7 58 26 PM</a></p> -
- - - - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-08 20:28:58
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/15|#15 New feature/banner images for lineage post>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-12 12:21:28
 -
-

[OpenLineage/OpenLineage] Issue opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/87|#87 Create OL FileBackend for testing>
-
- <p>To integration test the various OL integrations, we should create a FileBackend, such as exists in Marquez. This avoids a dependency on the Marquez implementation in the OpenLineage integration tests, so a bug in the implementation won't cause failures in the integration tests.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-12 12:30:22
 -
-

[OpenLineage/OpenLineage] Issue opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/88|#88 Migrate integration tests from Marquez to OpenLineage>
-
- <p>Port the integration tests for the airflow and spark integration tests to the OpenLineage project</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-12 12:32:26
 -
-

[OpenLineage/OpenLineage] Issue opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/89|#89 Update OpenLineage CI to publish to artifactory>
-
- <p>Unreleased packages should be deployed to<br /> -<a href="https://datakin.jfrog.io/artifactory/datakin-pypi/">https://datakin.jfrog.io/artifactory/datakin-pypi/</a> for testing</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-12 14:38:48
 -
-

[OpenLineage/OpenLineage] Issue opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/90|#90 Open Lineage integrations migration testing plan>
-
- <p>For the 0.1 OpenLineage release, we plan to migrate existing Marquez integrations to OpenLineage. In order to ensure backward compatiblity, we need a testing plan to validate the integration works seamlessly.</p> - -<p>The proposed plan is as follows</p> - -<ol><li>Migrate Marquez integration tests with a FileBackend implementation of the OpenLineage client- we want to avoid a dependency on the Marquez implementation causing test failures that are actually server-side bugs</li><li>Update CI scripts to publish OpenLineage pre-release artifacts to Artifactory- there's an endpoint at <a href="https://datakin.jfrog.io/artifactory/datakin-pypi/">https://datakin.jfrog.io/artifactory/datakin-pypi/</a> we can use for testing.</li><li>Update Marquez public integration classes to wrap OpenLineage implementations- e.g., <code>marquez_airflow.Dag</code> should wrap the <code>openlineage.airflow.Dag</code></li><li>Update Marquez CI to pull pre-release OpenLineage artifacts from artifactory</li><li>Run existing Marquez integration tests and validate the wrappers work with the underlying OpenLineage implementations.</li> -</ol> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-12 20:45:05
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/74|#74 Move integration/common from marquez>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-13 05:52:02
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/85|#85 dbt: parsing manifest>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-13 06:48:11
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/91|#91 notice file: change format to .txt>
-
- <p>Notice file is usually in .txt format.</p> - -<p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- -
-
Reviewers
- wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-13 11:46:07
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/91|#91 notice file: change format to .txt>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-13 12:07:23
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/92|#92 integrate openlineage-spark with the openlineage-java client>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-13 12:36:29
 -
-

[OpenLineage/website] Pull request opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/website/pull/16|#16 Fixed Spark integration docs to use correct configuration params>
-
- -
- -
-
Reviewers
- rossturk, wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-13 14:07:21
 -
-

[OpenLineage/website] Pull request merged by collado-mike

-
- - - - -
<https://github.com/OpenLineage/website/pull/16|#16 Fixed Spark integration docs to use correct configuration params>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-16 14:51:03
 -
-

[OpenLineage/OpenLineage] Pull request opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/94|#94 moving spark integration from https github com MarquezProject marquez>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - -
- - - } - - DCO: DCO - (https://github.com/OpenLineage/OpenLineage/runs/3089117573) -
- - - -
-
- -
- - - - - - - - -
-
- - - - -
-
- ✅ No checks have passed -
- - - - - - - -
- - 0/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-16 15:12:57
 -
-

[OpenLineage/OpenLineage] Pull request closed by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/94|#94 moving spark integration from https github com MarquezProject marquez>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-16 15:36:58
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/95|#95 Spark integration>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-19 09:02:13
 -
-

[OpenLineage/OpenLineage] Pull request opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/96|#96 Spark move with history>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - -
- - - } - - DCO: DCO - (https://github.com/OpenLineage/OpenLineage/runs/3104231996) -
- - - -
-
- -
- - - - - - - - -
-
- - - - -
-
- ✅ No checks have passed -
- - - - - - - -
- - 0/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-19 10:46:18
 -
-

[OpenLineage/OpenLineage] Pull request closed by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/95|#95 Spark integration>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-20 08:50:02
 -
-

[OpenLineage/OpenLineage] Pull request opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/97|#97 Spark move with history ci test>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - -
- - - } - - DCO: DCO - (https://github.com/OpenLineage/OpenLineage/runs/3114158522) -
- - - -
-
- -
- - - - - - - - -
-
- - - - -
-
- ✅ No checks have passed -
- - - - - - - -
- - 0/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-20 11:53:11
 -
-

[OpenLineage/OpenLineage] Issue opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/98|#98 Fix `spark.openlineage.url` handling>
-
- -
- -
-
Assignees
- OleksandrDvornik -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-20 12:03:16
 -
-

[OpenLineage/OpenLineage] Pull request closed by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/97|#97 Spark move with history ci test>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-21 10:39:13
 -
-

[OpenLineage/OpenLineage] Pull request opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/99|#99 openlineage#98 fix spark.openlineage.url>
-
- -
- -
-
Comments
- 1 -
- -
-
Reviewers
- wslulciuc, collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-21 18:05:44
 -
-

[OpenLineage/OpenLineage] Issue opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/100|#100 Rename `OpenLineageSparkContext` to `OpenLineageContext `>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-22 12:28:05
 -
-

[OpenLineage/OpenLineage] Pull request merged by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/96|#96 Spark move with history>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-22 15:10:45
 -
-

[OpenLineage/website] Pull request opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/website/pull/17|#17 Update fields in lineage blog to use correct names>
-
- <p>At some point before I merged my code changes, I changed these field names, but never updated the blog 🤦‍♂️:skin<em>tone</em>4:</p> -
- -
-
Reviewers
- rossturk -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Ross Turk - (ross@datakin.com) -
-
2021-07-22 16:26:39
 -
-

*Thread Reply:* I think I may try to deploy this change at the same time as a blog post about today’s LFAI announcement

- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-22 18:59:17
 -
-

[OpenLineage/website] Pull request merged by collado-mike

-
- - - - -
<https://github.com/OpenLineage/website/pull/17|#17 Update fields in lineage blog to use correct names>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-22 20:41:23
 -
-

[OpenLineage/website] Pull request opened by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/18|#18 Adding new blog about joining the LFAI>
-
- <p>Signed-off-by: Ross Turk <a href="mailto:ross@rossturk.com">ross@rossturk.com</a></p> - -<p>A preview image:<br /> -<a href="https://user-images.githubusercontent.com/1434475/126725528-d0128081-578a-43d8-91fa-ac515b766b1d.png">Screenshot 2021-07-22 at 20-37-41 OpenLineage joins the LF AI Data Foundation</a></p> -
- -
-
Reviewers
- julienledem -
- - - - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-22 21:08:30
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/18|#18 Adding new blog about joining the LFAI>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Julien Le Dem - (julien@apache.org) -
-
2021-07-22 21:37:23
 -
-

*Thread Reply:* Thanks. I was driving

- - - -
-
-
-
- - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-23 14:01:40
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/101|#101 Add the DataQualityMetrics Dataset facet to the core spec>
-
- <p><a href="https://github.com/MarquezProject/marquez/blob/main/integrations/airflow/marquez_airflow/data-quality-dataset-facet.json">https://github.com/MarquezProject/marquez/blob/main/integrations/airflow/marquez_airflow/data-quality-dataset-facet.json</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-23 14:02:54
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/102|#102 Add the OutputStatistics Run facet to the core spec>
-
- <p><a href="https://github.com/OpenLineage/OpenLineage/blob/main/integration/spark/facets/spark-2.4/v1/output-statistics-facet.json">https://github.com/OpenLineage/OpenLineage/blob/main/integration/spark/facets/spark-2.4/v1/output-statistics-facet.json</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-26 07:49:25
 -
-

[OpenLineage/OpenLineage] Pull request opened by fiskus

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/103|#103 Fix typo: and → an>
-
- <p>Hi, thank you for your work.<br /> -This is just a fix of a small typo I stumbled upon while reading docs.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-26 11:15:45
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/93|#93 move OpenLineage airflow integration to OpenLineage repository>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-26 13:32:02
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/104|#104 Import marquez-dbt integration to OpenLineage>
-
- -
- -
-
Assignees
- mobuchowski -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-26 14:46:17
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/105|#105 [PROPOSAL] package level job metadata>
-
- <p><strong>Purpose</strong></p> - -<p>To expose information like the version of the job and its source repository, we need a mechanism to provide metadata in the job package when deploying to a scheduler.<br /> -An example of this is deploying DAGs to an Airflow instance, but this is applicable to all package deployments in general.<br /> -In some instances workflows/jobs are directly pulled from a source control system from the scheduler allowing some metadata to be inferred from the environment. In other instances, the jobs are packaged and deployed to a scheduler (example: google cloud composer or astronomer). This doc concerns the later case of providing metadata related to the source code of the job at build time.</p> - -<p>(This is based on a Marquez proposal: <a href="https://github.com/MarquezProject/marquez/issues/825">MarquezProject/marquez#825</a>)</p> - -<p><strong>Proposal</strong></p> - -<p>We define a metadata file to be added at the root of the package/deployment location. This file is named <strong>.OpenLineage.job.facets.json</strong> and is a json file with the following initial properties (that can be expanded later).</p> - -<p>This also adds "repoUrl", "path", "version", "branch" to the existing SourceCodeLocation facet.</p> - -<p><a href="https://github.com/OpenLineage/OpenLineage/blob/6909a3da92ec69e464a90982f6b2c3727353955e/spec/OpenLineage.json#L339-L362">OpenLineage/spec/OpenLineage.json</a></p> - -<p>Lines 339 to 362 in &lt;/OpenLineage/OpenLineage/commit/6909a3da92ec69e464a90982f6b2c3727353955e|6909a3d&gt;</p> - -<p><strong>Properties:</strong></p> - -<p>• <strong>sourceCodeLocation.type</strong>: the source control system<br /> - Examples: GIT, SVN, ... -• <strong>sourceCodeLocation.url</strong>: the full http URL to locate the file<br /> - Example: <a href="https://github.com/MarquezProject/marquez-airflow-quickstart/blob/693e35482bc2e526ced2b5f9f76ef83dec6ec691/dags/dummy_example.py">https://github.com/MarquezProject/marquez-airflow-quickstart/blob/693e35482bc2e526ced2b5f9f76ef83dec6ec691/dags/dummy<em>example.py</a> -• <strong>sourceCodeLocation.repoUrl</strong>: the URL to the repository<br /> - Examples: - • GIT: <a href="mailto:git@github.com">git@github.com</a>:{org}/{repo}.git or <a href="https://github.com/%7Borg%7D/%7Brepo%7D.git|https://github.com/{org}/{repo}.git">https://github.com/%7Borg%7D/%7Brepo%7D.git|https://github.com/{org}/{repo}.git</a> - • SVN: svn://&lt;your</em>ip&gt;/&lt;repository_name&gt; -• <strong>sourceCodeLocation.path</strong>: the path in the repo containing the source files<br /> - Example: path/to/my/dags<br /> - It is intended that we can reconstruct a URL from that path pointing to the file defining a job. <a href="https://github.com/%7Borg%7D/%7Brepo%7D/tree/%7Bversion%7D/%7Bpath%7D/%7Bfile|https://github.com/{org}/{repo}/tree/{version}/{path}/{file">https://github.com/%7Borg%7D/%7Brepo%7D/tree/%7Bversion%7D/%7Bpath%7D/%7Bfile|https://github.com/{org}/{repo}/tree/{version}/{path}/{file</a> path in the package} -• <strong>sourceCodeLocation.version</strong>: the current version deployed (not a branch name, the actual unique version)<br /> - Examples: - • Git: the git sha - • Svn: the revision number -• <strong>sourceCodeLocation.tag</strong>: optional tag name<br /> - Examples: {tag name} -• <strong>sourceCodeLocation.branch</strong>:optional branch name</p> - -<p><strong>JSON Schema</strong></p> - -<pre><code>{ - "$schema": "&lt;https://json-schema.org/draft/2019-09/schema&gt;", - "type": "object", - "properties": { - "sourceCodeLocation": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "url": { - "type": "string" - }, - "repoUrl": { - "type": "string" - }, - "path": { - "type": "string" - }, - "version": { - "type": "string" - }, - "tag": { - "type": "string" - }, - "branch": { - "type": "string" - } - }, - "required": [ - "type", - "url" - ] - } - } -} - -</code></pre> - -<p><strong>Example:</strong></p> - -<p>For the DAG defined at: <a href="https://github.com/MarquezProject/marquez-airflow-quickstart/blob/693e35482bc2e526ced2b5f9f76ef83dec6ec691/dags/dummy_example.py">https://github.com/MarquezProject/marquez-airflow-quickstart/blob/693e35482bc2e526ced2b5f9f76ef83dec6ec691/dags/dummy_example.py</a></p> - -<pre><code>{ - "sourceCodeLocation": { - "type": "GIT", - "url": "&lt;https://github.com/MarquezProject/marquez-airflow-quickstart/blob/693e35482bc2e526ced2b5f9f76ef83dec6ec691/dags/dummy_example.py&gt;", - "repoUrl": "&lt;https://github.com/MarquezProject/marquez-airflow-quickstart.git&gt;", - "path": "dags", - "version": "693e35482bc2e526ced2b5f9f76ef83dec6ec691", - "branch": "master" - } -} - -</code></pre> -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-26 14:55:04
 -
-

[OpenLineage/OpenLineage] Issue opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/106|#106 The dbt `openlineage_bigquery` adapter does not support query cancellation>
-
- <p>Using the <code>openlineage_bigquery</code> adapter for OpenLineage throws the following error:</p> - -<pre><code>. -. -2021-07-22 00:25:58.454756 (MainThread): 17:25:58 | �[33mThe openlineage_bigquery adapter does not support query cancellation. Some queries may still be running!�[0m -2021-07-22 00:25:58.724444 (Thread-1): �[31mUnhandled error while executing model.******.******�[0m -404 DELETE &lt;https://bigquery.googleapis.com/bigquery/v2/projects/******/datasets/******/tables/******__dbt_tmp?prettyPrint=false&gt;: Not found: Table ******:******.******__dbt_tmp -2021-07-22 00:25:58.375309 (Thread-1): -Traceback (most recent call last): - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/dbt/task/base.py", line 348, in safe_run - result = self.compile_and_execute(manifest, ctx) - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/dbt/task/base.py", line 291, in compile_and_execute - result = self.run(ctx.node, manifest) - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/dbt/task/base.py", line 393, in run - return self.execute(compiled_node, manifest) - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/dbt/task/run.py", line 249, in execute - result = MacroGenerator(materialization_macro, context)() - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/dbt/clients/jinja.py", line 333, in __call__ - return self.call_macro(**args, ****kwargs) - File "/Users/max/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/dbt/clients/jinja.py", line 260, in call_macro - return macro(**args, ****kwargs) - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/jinja2/runtime.py", line 675, in __call__ - return self._invoke(arguments, autoescape) - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/jinja2/runtime.py", line 679, in _invoke - rv = self._func(**arguments) - File "&amp;lt;template&amp;gt;", line 44, in macro - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/jinja2/sandbox.py", line 462, in call - return __context.call(__obj, **args, ****kwargs) - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/jinja2/runtime.py", line 290, in call - return __obj(**args, ****kwargs) - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/dbt/adapters/bigquery/impl.py", line 159, in drop_relation - client.delete_table(relation_object) - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/google/cloud/bigquery/client.py", line 1717, in delete_table - timeout=timeout, - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/google/cloud/bigquery/client.py", line 685, in _call_api - return call() - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/google/api_core/retry.py", line 286, in retry_wrapped_func - on_error=on_error, - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/google/api_core/retry.py", line 184, in retry_target - return target() - File "/Users/******/.pyenv/versions/3.7.7/envs/dbt/lib/python3.7/site-packages/google/cloud/_http.py", line 438, in api_request - raise exceptions.from_http_response(response) -. - -</code></pre> - -<p>The issue might not be directly related to the query cancellation error, as in the logs, you can see an attempt for the <code>openlineage_bigquery</code> adapter to delete a tmp table that does not exist resulting in a <code>404</code> client exception. Note, the logs above were enabled via the <code>--debug</code> flag that redirects logs to <code>logs/dbt.log</code>. Note, when using the vanilla dbt <code>bigquery</code> adapter, no error is thrown. Below, you can find the related libraries used to reproduce the error:</p> - -<pre><code>bigquery==0.0.12 -dbt-bigquery==0.20.0 -google-cloud-bigquery==2.16.1 -google-cloud-bigquery-storage==2.4.0 -marquez-dbt-bigquery==0.16.1 - -</code></pre> -
- -
-
Assignees
- mobuchowski -
- -
-
Labels
- bug -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-26 15:03:09
 -
-

[OpenLineage/OpenLineage] Pull request opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/107|#107 add more detailed SourceCodeLocation fields>
-
- <p>Signed-off-by: Julien Le Dem <a href="mailto:julien@apache.org">julien@apache.org</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-26 16:49:35
 -
-

[OpenLineage/OpenLineage] Issue opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/108|#108 Add support for `pandas.read_sql`>
-
- <p>Let's add support for the Panda wrapper method <a href="https://pandas.pydata.org/docs/reference/api/pandas.read_sql.html"><code>read_sql</code></a>:</p> - -<pre><code>pandas.read_sql(sql, con, index_col=None, coerce_float=True, - params=None, parse_dates=None, columns=None, chunksize=None) - -</code></pre> - -<p>Since we have access to the <code>sql</code> at time of method execution, we can easily parse the SQL to capture the in / out tables before sending an OpenLineage event. We also have access to <code>columns</code> (if present) on table reads and may also want to support <code>read_sql_table</code> and <code>read_sql_query</code> (for backward compatibility).</p> -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-26 19:26:34
 -
-

[OpenLineage/OpenLineage] Pull request merged by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/103|#103 Fix typo: and → an>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 07:11:18
 -
-

[OpenLineage/OpenLineage] Pull request merged by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/99|#99 openlineage#98 fix spark.openlineage.url>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 07:12:20
 -
-

[OpenLineage/OpenLineage] Pull request opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/109|#109 Update OpenLineage CI to publish to artifactory #89>
-
- -
- -
-
Comments
- 1 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 08:00:18
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/110|#110 airflow integration: move to OpenLineage repo with history>
-
- -
- -
-
Reviewers
- wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - -
- - - } - - DCO: DCO - (https://github.com/OpenLineage/OpenLineage/runs/3171451768) -
- - - -
-
- -
- - - - - - - - -
-
- - - - -
-
- ✅ No checks have passed -
- - - - - - - -
- - 0/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 08:00:33
 -
-

[OpenLineage/OpenLineage] Pull request closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/93|#93 move OpenLineage airflow integration to OpenLineage repository>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 10:41:54
 -
-

[OpenLineage/OpenLineage] Issue closed by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/106|#106 The dbt `openlineage_bigquery` adapter does not support query cancellation>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 11:52:52
 -
-

[OpenLineage/OpenLineage] Issue opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/111|#111 dbt integration: create wrapper around dbt run >
-
- <p>Create wrapper around dbt run that parses manifest using <a href="https://github.com/OpenLineage/OpenLineage/pull/85/files">https://github.com/OpenLineage/OpenLineage/pull/85/files</a></p> - -<p>Should be python script that passes all arguments to actual dbt, and emits OL events to backends specified in env.</p> -
- -
-
Assignees
- mobuchowski -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 18:19:19
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/110|#110 airflow integration: move to OpenLineage repo with history>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 18:51:14
 -
-

[OpenLineage/OpenLineage] Pull request opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/113|#113 add Job naming convention>
-
- -
- -
-
Reviewers
- mandy-chessell, collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 19:05:50
 -
-

[OpenLineage/OpenLineage] Issue closed by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/84|#84 Finalize the OpenLineage mission statement>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 19:17:21
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/112|#112 Add `openlineage-dbt`>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
Julien Le Dem - (julien@apache.org) -
-
2021-07-27 19:55:17
 -
-

*Thread Reply:* I reverted this :face_palm: git push origin head

- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 19:53:34
 -
-

[OpenLineage/OpenLineage] Pull request opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/114|#114 add OutputStatistics facet>
-
- <p>Signed-off-by: Julien Le Dem <a href="mailto:julien@apache.org">julien@apache.org</a></p> -
- -
-
Comments
- 1 -
- -
-
Reviewers
- mobuchowski, OleksandrDvornik, collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 19:58:06
 -
-

[OpenLineage/website] Pull request opened by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/19|#19 New blog - extending OL with facets!>
-
- <p>Signed-off-by: Ross Turk <a href="mailto:ross@rossturk.com">ross@rossturk.com</a></p> - -<p>Screenshot attached:</p> - -<p><a href="https://user-images.githubusercontent.com/1434475/127242387-8361cf59-fed6-4345-8df0-fa4196666a0c.png">Screenshot 2021-07-27 at 19-56-50 Extending OpenLineage with Facets</a></p> -
- - - - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 19:58:59
 -
-

[OpenLineage/OpenLineage] Issue closed by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/105|#105 [PROPOSAL] package level job metadata>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 19:59:00
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/107|#107 add more detailed SourceCodeLocation fields>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 20:03:11
 -
-

[OpenLineage/website] Pull request merged by rossturk

-
- - - - -
<https://github.com/OpenLineage/website/pull/19|#19 New blog - extending OL with facets!>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-27 21:08:39
 -
-

[OpenLineage/OpenLineage] Pull request opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/115|#115 Data quality facet>
-
- -
- -
-
Reviewers
- mobuchowski, collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-28 10:07:57
 -
-

[OpenLineage/OpenLineage] Pull request opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/116|#116 adding new_version.sh script for bump up versions >
-
- -
- -
-
Reviewers
- wslulciuc, mobuchowski -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-28 12:25:23
 -
-

[OpenLineage/OpenLineage] Pull request merged by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/85|#85 dbt: parsing manifest>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-28 13:12:45
 -
-

[OpenLineage/OpenLineage] Pull request closed by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/112|#112 Add `openlineage-dbt`>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-28 18:14:51
 -
-

[OpenLineage/OpenLineage] Issue closed by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/104|#104 Import marquez-dbt integration to OpenLineage>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-28 18:44:54
 -
-

[OpenLineage/OpenLineage] Pull request opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/118|#118 separate facets>
-
- <p>Signed-off-by: Julien Le Dem <a href="mailto:julien@apache.org">julien@apache.org</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-29 09:45:25
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/117|#117 dbt: wrapper script>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-29 09:47:31
 -
-

[OpenLineage/OpenLineage] Pull request merged by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/116|#116 adding new_version.sh script for bump up versions >
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-29 12:39:38
 -
-

[OpenLineage/OpenLineage] Pull request opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/119|#119 spark: rename OpenLineageSparkContext to OpenLineageContext>
-
- -
- -
-
Comments
- 1 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-29 12:44:01
 -
-

[OpenLineage/OpenLineage] Issue opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/120|#120 SchemaDatasetFacet: relax type requirement>
-
- <p>Right now, type property for schema field is required:<br /> -<a href="https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json#L462">https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json#L462</a></p> - -<p>It's easy to imagine cases where this restriction disallows us from emitting useful information. For example, <code>dbt</code> metadata sometimes does not always emit types. Another example might be etl job that selects subset of columns - it would be nice to see which ones are selected, even if we don't know their types.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-29 13:09:24
 -
-

[OpenLineage/OpenLineage] Pull request merged by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/119|#119 spark: rename OpenLineageSparkContext to OpenLineageContext>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-29 19:41:02
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/121|#121 Add version id in OpenLineage spec and publish on website>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-30 08:29:37
 -
-

[OpenLineage/OpenLineage] Pull request opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/122|#122 change new-version.sh according to comments>
-
- -
- -
-
Reviewers
- wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-30 09:39:36
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/123|#123 SchemaDatasetFacet: do not require field type >
-
- <p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- -
-
Reviewers
- julienledem, collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-30 14:11:10
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/122|#122 change new-version.sh according to comments>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-30 15:07:07
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/123|#123 SchemaDatasetFacet: do not require field type >
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-30 15:20:27
 -
-

[OpenLineage/OpenLineage] Pull request opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/124|#124 add version in spec>
-
- <p>Signed-off-by: Julien Le Dem <a href="mailto:julien@apache.org">julien@apache.org</a></p> -
- -
-
Reviewers
- collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-30 17:11:47
 -
-

[OpenLineage/OpenLineage] Issue opened by knxacgcg

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/125|#125 Date doesn't parse correctly when a Pendulum date is used>
-
- <p><a href="https://github.com/OpenLineage/OpenLineage/blob/0a6be73d855e9bd5e1c61453190343b17ca1082d/integration/airflow/openlineage/airflow/utils.py#L216-L217">OpenLineage/integration/airflow/openlineage/airflow/utils.py</a></p> - -<p>Lines 216 to 217 in &lt;/OpenLineage/OpenLineage/commit/0a6be73d855e9bd5e1c61453190343b17ca1082d|0a6be73&gt;</p> - -<p>Example:</p> - -<pre><code>&amp;gt;&amp;gt;&amp;gt; _NOMINAL_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" -&amp;gt;&amp;gt;&amp;gt; -&amp;gt;&amp;gt;&amp;gt; import pendulum -&amp;gt;&amp;gt;&amp;gt; dt = pendulum.now("America/Los_Angeles") -&amp;gt;&amp;gt;&amp;gt; dt.format(_NOMINAL_TIME_FORMAT) -'%2021-%8-%5T%14:%7:%6.%f-07:00' -</code></pre> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-07-30 19:42:42
 -
-

[OpenLineage/OpenLineage] Issue opened by knxacgcg

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/126|#126 create_dagrun doesn't seem to work when called from scheduler in airflow 2.0>
-
- <p><a href="https://github.com/OpenLineage/OpenLineage/blob/0a6be73d855e9bd5e1c61453190343b17ca1082d/integration/airflow/openlineage/airflow/dag.py#L102">OpenLineage/integration/airflow/openlineage/airflow/dag.py</a></p> - -<p>Line 102 in &lt;/OpenLineage/OpenLineage/commit/0a6be73d855e9bd5e1c61453190343b17ca1082d|0a6be73&gt;</p> - -<p><strong>Dependencies:</strong></p> - -<p>Airflow Version: 2.1.2<br /> -Python Version: 3.8.0<br /> -OpenLineage Library Versions:</p> - -<p>• git+<a href="https://github.com/OpenLineage/OpenLineage.git@47fdac36d32f607e611e4f7838637e294929518f#egg=openlineage-python&amp;subdirectory=client/python|https://github.com/OpenLineage/OpenLineage.git@47fdac36d32f607e611e4f7838637e294929518f#egg=openlineage-python&amp;subdirectory=client/python">https://github.com/OpenLineage/OpenLineage.git@47fdac36d32f607e611e4f7838637e294929518f#egg=openlineage-python&amp;subdirectory=client/python|https://github.com/OpenLineage/OpenLineage.git@47fdac36d32f607e611e4f7838637e294929518f#egg=openlineage-python&amp;subdirectory=client/python</a> -• git+<a href="https://github.com/OpenLineage/OpenLineage.git@47fdac36d32f607e611e4f7838637e294929518f#egg=openlineage-integration-common&amp;subdirectory=integration/common|https://github.com/OpenLineage/OpenLineage.git@47fdac36d32f607e611e4f7838637e294929518f#egg=openlineage-integration-common&amp;subdirectory=integration/common">https://github.com/OpenLineage/OpenLineage.git@47fdac36d32f607e611e4f7838637e294929518f#egg=openlineage-integration-common&amp;subdirectory=integration/common|https://github.com/OpenLineage/OpenLineage.git@47fdac36d32f607e611e4f7838637e294929518f#egg=openlineage-integration-common&amp;subdirectory=integration/common</a> -• git+<a href="https://github.com/OpenLineage/OpenLineage.git@47fdac36d32f607e611e4f7838637e294929518f#egg=openlineage-airflow&amp;subdirectory=integration/airflow|https://github.com/OpenLineage/OpenLineage.git@47fdac36d32f607e611e4f7838637e294929518f#egg=openlineage-airflow&amp;subdirectory=integration/airflow">https://github.com/OpenLineage/OpenLineage.git@47fdac36d32f607e611e4f7838637e294929518f#egg=openlineage-airflow&amp;subdirectory=integration/airflow|https://github.com/OpenLineage/OpenLineage.git@47fdac36d32f607e611e4f7838637e294929518f#egg=openlineage-airflow&amp;subdirectory=integration/airflow</a></p> - -<p><strong>Sample DAG:</strong></p> - -<pre><code>from datetime import datetime - -from openlineage.airflow import DAG -from airflow.operators.bash import BashOperator -import http -http.client.HTTPConnection.debuglevel = 1 - -airflow_defaults = {"owner": "test-airflow", "start_date": datetime(2020, 8, 19)} -dag = DAG( - "sample_run_1", - default_args=airflow_defaults, - catchup=False, - schedule_interval="** ** ** ** **" -) - -with dag: - bash_task = BashOperator( - task_id="bash_task", - bash_command=""" echo "${OPENLINEAGE_URL} | ${OPENLINEAGE_PRODUCER} | ${OPENLINEAGE_API_KEY} | ${OPENLINEAGE_NAMESPACE}" """ - ) -</code></pre> - -<p><strong>Runs</strong></p> - -<ol><li>Running via Interpreter:<br /> -when running directly through a python script, the openlineage override create_dagrun seems to work correctly and has the correct type. it also properly emits to the openlineage endpoint</li> -</ol> - -<pre><code>from airflow.utils.types import DagRunType -from airflow.utils.state import State -from datetime import datetime -from airflow.utils import timezone - -dag.create_dagrun(state=State.QUEUED, run_type=DagRunType.MANUAL, execution_date=timezone.make_aware(datetime.now())) -</code></pre> - -<pre><code>[2021-07-30 16:28:28,412] {dag.py:103} INFO - ******** STARTING CREATE DAGRUN OPENLINEAGE ************ -[2021-07-30 16:28:28,413] {dag.py:1770} INFO - ******** STARTING CREATE DAGRUN ************ -[2021-07-30 16:28:28,413] {dag.py:1771} INFO - &amp;lt;DAG: sample_run_1&amp;gt;. &amp;lt;class 'openlineage.airflow.dag.DAG'&amp;gt; -</code></pre> - -<ol start="2"><li>Running via Scheduler:<br /> -however, when triggered from the scheduler / ui, it appears to only run the native create_dagrun method and nothing is emitted to the openlineage endpoint</li> -</ol> - -<pre><code>[2021-07-30 16:29:02,426] {dag.py:1770} INFO - ******** STARTING CREATE DAGRUN ************ -[2021-07-30 16:29:02,426] {dag.py:1771} INFO - &amp;lt;DAG: sample_run_1&amp;gt;. &amp;lt;class 'airflow.serialization.serialized_objects.SerializedDAG'&amp;gt; -</code></pre> - -<p><strong>How to reproduce:</strong></p> - -<p>• create a log entry ( or debug breakpoint ) within openlineage.airflow.dag.DAG.create<em>dagrun and airflow.models.dag.DAG.create</em>dagrun -• run a dag that uses openlineage.airflow.DAG -• check if both log entries appear, or if only the log entry from airflow.models.dag.DAG.create<em>dagrun is emitted. additionally, there will be no request made to the OPENLINEAGE</em>URL endpoint</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-02 05:30:04
 -
-

[OpenLineage/OpenLineage] Issue opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/127|#127 dbt integration: use data from catalog.json when available >
-
- <p>When user runs <code>dbt docs generate</code> dbt does grab - and store in <code>catalog.json</code> file - additional table and column level metadata, such as</p> - -<pre><code>Column type -Approximate size of table -Approximate count of rows in this table - -</code></pre> - -<p>We should use those.</p> -
- -
-
Assignees
- mobuchowski -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-02 05:34:49
 -
-

[OpenLineage/OpenLineage] Pull request merged by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/117|#117 dbt: wrapper script>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-02 05:35:17
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/120|#120 SchemaDatasetFacet: relax type requirement>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-02 05:39:29
 -
-

[OpenLineage/OpenLineage] Issue closed by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/100|#100 Rename `OpenLineageSparkContext` to `OpenLineageContext `>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-02 12:20:24
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/128|#128 Add automated tests for Spark integration with Spark 3.x in addition to Spark 2.4>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-03 10:29:59
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/129|#129 airflow: remove test dependency on marquez>
-
- <p>Replace comparing generated events via Marquez with comparison to file-provided events.<br /> -Comparison is done by simple flask service.</p> - -<p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- -
-
Reviewers
- wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-03 10:50:17
 -
-

[OpenLineage/OpenLineage] Pull request opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/130|#130 [spark-client-integration] changes in java client for spark integration>
-
- <p>Changes required for <a href="https://github.com/OpenLineage/OpenLineage/pull/131">PR</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-03 11:15:16
 -
-

[OpenLineage/OpenLineage] Pull request opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/131|#131 [spark-client-integration] spark part, use new changes from openlineage-java client>
-
- <p>Depends on <a href="https://github.com/OpenLineage/OpenLineage/pull/130">PR</a><br /> -Java client should be published to snapshot, before merge this PR.<br /> -Test can fail due to dependency and latest changes in openlineage-java</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-03 11:30:26
 -
-

[OpenLineage/OpenLineage] Pull request opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/132|#132 Artifactory test ci>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-03 11:34:53
 -
-

[OpenLineage/OpenLineage] Issue closed by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/63|#63 [PROPOSAL] Define the OpenLineage spec versioning mechanism>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-03 11:34:53
 -
-

[OpenLineage/OpenLineage] Issue closed by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/121|#121 Add version id in OpenLineage spec and publish on website>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-03 11:34:54
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/124|#124 add version in spec>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-03 12:21:46
 -
-

[OpenLineage/OpenLineage] Issue opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/133|#133 README for the DBT integration>
-
- -
- -
-
Assignees
- mobuchowski -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-03 12:29:35
 -
-

[OpenLineage/OpenLineage] Pull request closed by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/132|#132 Artifactory test ci>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-03 13:32:02
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/134|#134 dbt: readme>
-
- <p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- -
-
Reviewers
- wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-03 17:58:41
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/114|#114 add OutputStatistics facet>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-03 17:59:12
 -
-

[OpenLineage/OpenLineage] Issue closed by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/102|#102 PROPOSAL: Add the OutputStatistics Output Dataset facet to the core spec>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 05:59:21
 -
-

[OpenLineage/OpenLineage] Pull request merged by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/130|#130 [spark-client-integration] changes in java client for spark integration>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 10:27:57
 -
-

[OpenLineage/OpenLineage] Pull request merged by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/109|#109 Update OpenLineage CI to publish to artifactory #89>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Oleksandr Dvornik - (oleksandr.dvornik@getindata.com) -
-
2021-08-04 10:42:12
 -
-

@Oleksandr Dvornik has joined the channel

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 11:06:12
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/135|#135 dbt wrapper: fix dependencies>
-
- <p>• properly refer to integration-common -• remove transient dependencies -• force sqlparse &lt; 4 when using dbt script</p> -
- -
-
Reviewers
- wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 11:44:24
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/135|#135 dbt wrapper: fix dependencies>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 12:43:36
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/136|#136 python: remove usage of OPENLINAGE_PRODUCER environment variable>
-
- <p>Integrations can change default producer using <code>set_producer</code>. This is effectively singleton.</p> - -<p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- -
-
Reviewers
- wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Joe Regensburger - (jregensburger@immuta.com) -
-
2021-08-04 13:33:45
 -
-

@Joe Regensburger has joined the channel

- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 14:38:35
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/134|#134 dbt: readme>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 18:20:53
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/136|#136 python: remove usage of OPENLINAGE_PRODUCER environment variable>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 20:01:21
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/129|#129 airflow: remove test dependency on marquez>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 20:12:17
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/138|#138 Separate facets to allow versioning of the core spec and each facet independently>
-
- <p>We want to version the core spec (LineageEvent, Job, Run, Dataset) independently of all the facets.<br /> -Each facet gets its own version number by following the same versioning scheme as the spec.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 20:14:17
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/139|#139 Add automated tests for Airflow 2.0>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 22:49:20
 -
-

[OpenLineage/OpenLineage] Pull request opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/140|#140 Add mechanism to enforce spec versioning>
-
- <p>This modifies the code generator to:</p> - -<p>• always use the local version of the spec -• issue a warning if the spec is not public at $id -• verify that the spec matches the one published at $id if available -• fail the build if the spec does not match to require changing the version.</p> - -<p>This will guarantee that we don't reuse published specs.<br /> -It also removes the need for the code to be pushed to the repo for the build to work.</p> - -<p>Signed-off-by: Julien Le Dem <a href="mailto:julien@apache.org">julien@apache.org</a></p> -
- -
-
Comments
- 1 -
- -
-
Reviewers
- mobuchowski, OleksandrDvornik, collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 23:11:04
 -
-

[OpenLineage/OpenLineage] Issue closed by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/101|#101 Add the DataQualityMetrics Dataset facet to the core spec>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-04 23:11:05
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/115|#115 Data quality Metrics facet>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 05:12:56
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/141|#141 Rename Great Expectation assert property `columnId` -&gt; `column`>
-
- <p>This PR fixes the property name for the Great Expectation facet used in assertions from <code>columnId</code> to <code>column</code>. The property name in Great Expectation is <code>column</code> and we should keep names the same, see <a href="https://github.com/great-expectations/great_expectations/blob/main/great_expectations/core/expectation_configuration.py#L996">https://github.com/great-expectations/great<em>expectations/blob/main/great</em>expectations/core/expectation_configuration.py#L996</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 05:22:48
 -
-

[OpenLineage/OpenLineage] Issue opened by fm100

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/142|#142 `pip install openlineage-dbt` not working>
-
- <pre><code>(py39) ➜ ~ python --version -Python 3.9.6 -(py39) ➜ ~ pip install openlineage-dbt -ERROR: Could not find a version that satisfies the requirement openlineage-dbt (from versions: none) -ERROR: No matching distribution found for openlineage-dbt -(py39) ➜ ~ - -</code></pre> - -<p>Tried as instructed from dbt integration README, but <em>pip install</em> command didn't work. Tested on python 3.6, 3.7, 3.8 and 3.9 and nothing worked.</p> -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 12:04:23
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/137|#137 dbt: use column-level catalog metadata>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 14:03:59
 -
-

[OpenLineage/OpenLineage] New release Release - 0.0.1-rc7 published by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/releases/tag/0.0.1-rc7|Release - 0.0.1-rc7>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 14:04:10
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/143|#143 Add outputStatistics for spark integration>
-
- <p>In addition to the current outputStatistics dataset.facet facet we need to add the same information in <code>event.outputs.Dataset.outputFacets.outputStatistics</code></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 14:04:18
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/144|#144 Add outputStatistics for airflow-bq integration>
-
- <p>In addition to the current outputStatistics dataset.facet facet we need to add the same information in <code>event.outputs.Dataset.outputFacets.outputStatistics</code></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 14:04:31
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/145|#145 Add outputStatistics for dbt integration>
-
- <p>We need to add the outputStatistics information in <code>event.outputs.Dataset.outputFacets.outputStatistics</code></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 14:05:58
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/146|#146 Add dataQualityMetrics for airflow-GreatExpectations integration>
-
- <p>In addition to the current dataset.facets.dataQuality facet we need to add the same information in <code>event.inputs.Dataset.inputFacets.dataQualityMetrics</code></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 14:11:09
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/147|#147 Publish automatically the spec to openlineage.io>
-
- <p>When the build runs on the <code>main</code> branch, we will publish the <code>spec/OpenLineage.json</code> file to <a href="https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json#L3">the URL defined in the <code>$id</code> field</a> using <a href="https://github.com/OpenLineage/OpenLineage.github.io">github pages</a>..</p> - -<p>See versioning specification: <a href="https://github.com/OpenLineage/OpenLineage/blob/main/spec/Versioning.md">https://github.com/OpenLineage/OpenLineage/blob/main/spec/Versioning.md</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 14:45:05
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/148|#148 [PROPOSAL] column level lineage facet>
-
- <p><strong>Purpose:</strong><br /> -For transformations like SQL queries (but also in other cases), we can extract column level lineage. This allows answering questions like ‘which root input columns are used to construct column x?’. We need to be able to capture this information in the OpenLineage model</p> - -<p><strong>Proposed implementation</strong><br /> -We propose to introduce a new dataset facet:<br /> -example:</p> - -<pre><code>{ - "eventType": "COMPLETE", - "eventTime": "2020-12-28T20:52:00.001+10:00", - "run" : { - "runId": "uuid" - }, - "job": { - "namespace": "scheduler", - "name": "myjob", - "facets": { - "sql": { - "query": "Insert into outputTable from select ** from inputTable" - } - } - }, - "inputs": [ - { - "namespace": "N1", - "name": "inputTable", - "facets": { - "schema": { - "fields": [ {"name": "col_a", "type": "VARCHAR"}, {"name": "col_b", "type": "int"}] - } - } - } - ], - "outputs": [ - { - "namespace": "N2", - "name": "outputTable", - "facets": { - "schema": { - "fields": [ {"name": "col_a", "type": "VARCHAR"}, {"name": "col_b", "type": "int"}] - }, - "columnLineage": { - "fields": { - "col_a": [ { "namespace": "N1", "name": "col_a" } ], - "col_b": [ { "namespace": "N1", "name": "col_b" } ] - } - } - } - } - ] -} -</code></pre> - -<p>Schema:</p> - -<pre><code> "columnLineage": { - "type": "object", - "properties": { - "fields": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "namespace": { - "type": "string" - }, - "name": { - "type": "string" - }, - }, - "required": [ - "namespace", "name" - ] - } - } - } - } -</code></pre> -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 15:04:34
 -
-

[OpenLineage/OpenLineage] New release Release - 0.0.1-rc7 published by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/releases/tag/0.0.1-rc7|Release - 0.0.1-rc7>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 15:12:16
 -
-

[OpenLineage/OpenLineage] New release Release - 0.0.1-rc7 published by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/releases/tag/0.0.1-rc7|Release - 0.0.1-rc7>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 15:16:52
 -
-

[OpenLineage/OpenLineage] New release Release - 0.0.1-rc8 published by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/releases/tag/0.0.1-rc8|Release - 0.0.1-rc8>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-05 15:19:52
 -
-

[OpenLineage/OpenLineage] New release Release - 0.0.1-rc8 published by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/releases/tag/0.0.1-rc8|Release - 0.0.1-rc8>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-06 08:06:00
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/137|#137 dbt: use column-level catalog metadata>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-06 08:06:33
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/72|#72 Drop env var `OPENLINEAGE_PRODUCER` usage in py client>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-06 08:26:27
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/133|#133 README for the DBT integration>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-06 08:48:31
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/87|#87 Update integration tests to use mock-server container>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-06 08:48:46
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/88|#88 Migrate integration tests from Marquez to OpenLineage>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-06 09:04:26
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/111|#111 dbt integration: create wrapper around dbt run >
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-06 12:46:34
 -
-

[OpenLineage/OpenLineage] Pull request merged by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/141|#141 Rename Great Expectation assert property `columnId` -&gt; `column`>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-06 19:34:30
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/140|#140 Add mechanism to enforce spec versioning>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-09 09:22:51
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/150|#150 great expectations: data quality input facet>
-
- <p><a href="https://github.com/OpenLineage/OpenLineage/issues/146">#146</a><br /> -Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- -
-
Reviewers
- wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-09 10:15:56
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/151|#151 bigquery: output statistics facet>
-
- <p><a href="https://github.com/OpenLineage/OpenLineage/issues/144">#144</a><br /> -Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-09 11:43:19
 -
-

[OpenLineage/OpenLineage] Issue opened by mandy-chessell

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/152|#152 Proposal for a ProxyBackend>
-
- <p>This issue is to track the discussion and development of a proxy backend. The diagram below shows the basic idea.</p> - -<p><a href="https://user-images.githubusercontent.com/12354195/128734055-2a08372e-8d1b-426e-9c25-c06aeb551e36.png">image</a></p> -
- - - - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-09 16:12:10
 -
-

[OpenLineage/website] Pull request opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/website/pull/20|#20 Add new blog on data quality facets and Great Expectations>
-
- <p>Signed-off-by: Michael Collado <a href="mailto:mike@datakin.com">mike@datakin.com</a></p> -
- -
-
Reviewers
- julienledem, rossturk -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-09 19:38:47
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/153|#153 [PROPOSAL] Version facets independently of the core spec>
-
- <p><strong>Purpose:</strong><br /> -The OpenLineage core spec is versioned using a version URL in the <code>$id</code> field:<br /> -See:</p> - -<p>• <a href="https://github.com/OpenLineage/OpenLineage/blob/main/spec/Versioning.md">https://github.com/OpenLineage/OpenLineage/blob/main/spec/Versioning.md</a> -• <a href="https://openlineage.io/spec/1-0-0/OpenLineage.json">https://openlineage.io/spec/1-0-0/OpenLineage.json</a></p> - -<pre><code>{ - "$schema": "&lt;https://json-schema.org/draft/2020-12/schema&gt;", - "$id": "&lt;https://openlineage.io/spec/1-0-0/OpenLineage.json&gt;", -</code></pre> - -<p>Currently all the core facets are also defined in that same core spec.</p> - -<p>This means that the core spec gets a new version whenever a facet is changed along with every other facet.<br /> -We'd rather have the core spec and each facet versioned independently so that their version changes only when they change.<br /> -This matches much better the use cases for using the version number.</p> - -<p>For example:</p> - -<p>• a consumer of OpenLineage would care only about the compatibility of the core spec, accept any version of any facet and handles specific facets independently. -• a producer of a facet cares what version of the core model and that specific facet (core or custom) it is generating. -• a consumer of a specific facet cares about the version of that specific facet and ignore changes to facets it does not consume.</p> - -<p><strong>Proposed implementation</strong></p> - -<p>This proposal is to move each core facet definition into its own file and use the same versioning mechanism (per file) as the core spec.</p> - -<p>This leaves the core spec to only contain the <code>LineageEvent</code> and related core entities (LineageEvent, Run, Job, Dataset) but none of the core facets.</p> - -<p>See for example:<br /> -<a href="https://github.com/OpenLineage/OpenLineage/blob/789b55fc6e6e07d0aedbe130f775721829e83618/spec/OpenLineage.json">https://github.com/OpenLineage/OpenLineage/blob/789b55fc6e6e07d0aedbe130f775721829e83618/spec/OpenLineage.json</a><br /> -This would be the spec that a consumer would implement to accept any facet version.<br /> -This would ensure that the core spec version changes rarely, only when we add fields to the core model.<br /> -Each facet would be versioned independently following the same versioning mechanism. Each facet version will change if and only if this particular facet schema changes. It also makes core facets and custom facets work exactly the same, they are just maintained in different repositories.</p> - -<p>Example:<br /> -<a href="https://openlineage.io/spec/1-0-0/DatasetSchemaFacet.json">https://openlineage.io/spec/1-0-0/DatasetSchemaFacet.json</a></p> - -<pre><code>{ - "$schema": "&lt;https://json-schema.org/draft/2020-12/schema&gt;", - "$id": "&lt;https://openlineage.io/spec/1-0-0/DatasetSchemaFacet.json&gt;", - "type": "object", - "properties": { - "schema": { - "allOf": [ - { - "$ref": "&lt;https://openlineage.io/spec/1-0-0/OpenLineage.json#/definitions/DatasetFacet&gt;" - }, - { - "type": "object", - "properties": { - "fields": { - "description": "The fields of the table.", - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "description": "The name of the field.", - "type": "string", - "example": "column1" - }, - "type": { - "description": "The type of the field.", - "type": "string", - "example": "VARCHAR|INT|..." - }, - "description": { - "description": "The description of the field.", - "type": "string" - } - }, - "required": [ - "name", - "type" - ] - } - } - } - } - ], - "type": "object" - }, - } - } -} -</code></pre> -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 07:55:28
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/154|#154 release,ci: add dbt integration script>
-
- <p>Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- -
-
Reviewers
- wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 13:02:18
 -
-

[OpenLineage/OpenLineage] Issue opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/155|#155 [PROPOSAL] Add support for CloudSQL endpoint discovery/reporting>
-
- <p><strong>Purpose:</strong><br /> -Accessing CloudSQL resources generally involves using a local proxy that does the authentication with the remote database endpoint. Database drivers will then create a URL like <code>&lt;postgres://localhost:5432/my_database&gt;</code>, which the OpenLineage integration will always report as <code>localhost</code> in the data source metadata.</p> - -<p>It won't be possible to detect the actual database location from the OpenLineage integration directly, but we can probably add some configuration option so that users can report the correct database location.</p> - -<p><strong>Proposed implementation</strong><br /> -Add configuration option for user to report actual CloudSQL database location</p> -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 13:18:11
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/154|#154 release,ci: add dbt integration script>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 13:32:15
 -
-

[OpenLineage/website] Pull request merged by collado-mike

-
- - - - -
<https://github.com/OpenLineage/website/pull/20|#20 Add new blog on data quality facets and Great Expectations>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/website">OpenLineage/website</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 14:01:32
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/151|#151 bigquery: output statistics facet>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 14:10:57
 -
-

[OpenLineage/OpenLineage] Issue closed by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/144|#144 Add outputStatistics for airflow-bq integration>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 14:11:40
 -
-

[OpenLineage/OpenLineage] Issue closed by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/29|#29 [PROPOSAL] Make job run ids unique >
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 14:13:42
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/150|#150 great expectations: data quality input facet>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 14:16:44
 -
-

[OpenLineage/OpenLineage] Issue closed by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/57|#57 Import marquez-spark integration to OpenLineage integrations>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 20:10:51
 -
-

[OpenLineage/OpenLineage] Pull request opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/156|#156 Updated circleci to publish non-released python libs to artifactory>
-
- <p>Adds a copy of the existing <code>release-python</code> step with a publish to a new artifactory endpoint</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 20:25:57
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/157|#157 [Airflow] add support to enable the spark integration automatically when using spark operators>
-
- <p>Currently to use the spark integration in airflow, one has to change their DAG to use the listener:</p> - -<pre><code>t1 = DataProcPySparkOperator( - task_id=job_name, - gcp_conn_id='google_cloud_default', - project_id='project_id', - cluster_name='cluster-name', - region='us-west1', - main='&lt;gs://bucket/your-prog.py&gt;', - job_name=job_name, - dataproc_pyspark_properties={ - "spark.extraListeners": "marquez.spark.agent.SparkListener", - "spark.jars.packages": "io.github.marquezproject:marquez_spark:0.15.+", - "spark.openlineage.host": "{your openlineage consumer}", - "spark.openlineage.namespace": job_namespace, - "spark.openlineage.parentRunId": "{{task_run_id(run_id, task)}}", # airflow macro for reading the task's run id - }, - dag=dag) - -</code></pre> - -<p>We'd want the airflow integration to automatically enable it.<br /> -If extra configuration is needed, it should be done in a centralize way and not in the dag.</p> -
- -
-
Labels
- enhancement -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 20:31:36
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/137|#137 dbt: use column-level catalog metadata>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 20:33:31
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/156|#156 Updated circleci to publish non-released python libs to artifactory>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 20:35:01
 -
-

[OpenLineage/OpenLineage] Issue closed by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/127|#127 dbt integration: use data from catalog.json when available >
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-10 20:35:33
 -
-

[OpenLineage/OpenLineage] Issue closed by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/82|#82 Add dbt support>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 11:10:18
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/158|#158 Add guidelines on pull requests and branching>
-
- <p>This PR adds guidelines on submitting pull requests, signing a commit, and branching.</p> -
- -
-
Reviewers
- julienledem -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 11:36:48
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/159|#159 Backfill multi-statement sql parsing bug>
-
- <p>This PR backfills the multi-statement sql parsing bug, see <a href="https://github.com/MarquezProject/marquez/pull/1494">MarquezProject/marquez#1494</a></p> -
- -
-
Comments
- 1 -
- -
-
Reviewers
- julienledem -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 11:44:36
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/159|#159 Backfill multi-statement sql parsing bug>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 13:52:10
 -
-

[OpenLineage/OpenLineage] Pull request merged by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/131|#131 [spark-client-integration] spark part, use new changes from openlineage-java client>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 14:38:57
 -
-

[OpenLineage/OpenLineage] Pull request opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/160|#160 Added original OutputStatisticFacet back to Dataset facets for backward compatibility>
-
- <p>Signed-off-by: Michael Collado <a href="mailto:mike@datakin.com">mike@datakin.com</a></p> -
- -
-
Reviewers
- julienledem -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 14:50:46
 -
-

[OpenLineage/OpenLineage] Issue closed by fm100

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/142|#142 `pip install openlineage-dbt` not working>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 16:08:11
 -
-

[OpenLineage/OpenLineage] Pull request merged by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/160|#160 Added original OutputStatisticFacet back to Dataset facets for backward compatibility>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
- ❤️ Julien Le Dem -
- -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 16:25:48
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/161|#161 Add lfaidata project badge and status>
-
- <p>This PR adds the LF AI &amp; Data sandbox badge.</p> -
- -
-
Reviewers
- julienledem -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 16:35:44
 -
-

[OpenLineage/OpenLineage] Issue closed by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/66|#66 Clarify the job naming strategy>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 16:35:46
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/113|#113 add Job naming convention>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 16:43:39
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/162|#162 Add openlineage logo and badges section>
-
- <p>This PR adds the openlineage logo and a badges section in <code>README.md</code></p> -
- -
-
Reviewers
- julienledem -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 17:40:51
 -
-

[OpenLineage/OpenLineage] Pull request opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/163|#163 Fix snapshot builds>
-
- <p>Fixes python development builds to append <code>.dev${CIRCLE_BUILD_NUM}</code> to snapshot releases.</p> - -<p>I tested by temporarily adding support for the <code>dev[0-9]</code> tag regex and pushing the tag <code>dev1</code>. I'll delete the tag, but the source can be viewed at <a href="https://github.com/OpenLineage/OpenLineage/commit/265e790f39684260df7d977eeabe8cbabe1c18a0">265e790</a></p> -
- -
-
Comments
- 3 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 17:57:48
 -
-

[OpenLineage/OpenLineage] Pull request merged by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/163|#163 Fix snapshot builds>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 18:01:03
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/162|#162 Add openlineage logo and badges section>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 19:17:23
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/164|#164 [INTEGRATION] Add support for Trino>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 19:17:27
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/165|#165 [INTEGRATION] Add support for PrestoDB>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 19:21:29
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/166|#166 [INTEGRATION] In the Spark integration: Enable datasources (for example Iceberg) to emit facets to be attached to input/output datasets>
-
- <p>A Spark datasource like Iceberg should be able to emit Dataset facets (example file count written, time it took to commit, number of attempts) that will be added to the OpenLineage event without having to modify Spark APIs</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 19:23:16
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/167|#167 [INTEGRATION] Iceberg should be able to send OpenLineage events independently from where the library is used (including in a plain java process)>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 19:27:06
 -
-

[OpenLineage/OpenLineage] Issue opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/168|#168 [PROPOSAL] Making the spark integration extensible>
-
- <p>Ported from <a href="https://github.com/MarquezProject/marquez/issues/1192">MarquezProject/marquez#1192</a> . It's been updated to reflect the current status of the Spark integration.</p> - -<p><strong>Proposal</strong></p> - -<p>I propose a more extensible method of loading and discovering node visitors. The basic strategy is</p> - -<ol><li>Create a common interface for visiting logical plan nodes- something that extends the <code>PartitalFunction&amp;lt;LogicalPlan, List&amp;lt;Dataset&amp;gt;&amp;gt;</code> interface we already use</li><li>Take advantage of the JVM's <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html">ServiceLoader</a> capabilities to enable implementations on the classpath to be found at runtime</li><li>Package node visitors in different subprojects that generate their own jars</li><li>Entrust Spark users to declare dependencies on the node visitors they need using the <code>spark.jars.packages</code> or <code>spark.jars</code> configuration properties</li> -</ol> - -<p>The main advantages I see here are</p> - -<ol><li>Static typing, easy to read code, and easy unit testing</li><li>Extensible framework to support new integrations<br /> -This point, I feel, is extremely important. While the single jar deployment is convenient, it creates an unnecessary barrier for users who want to take part in the OpenLineage ecosystem, but can't/won't contribute directly to ~Marquez or~ OpenLineage. This may be because</li> -</ol> - -<p>• their company has strict open source contribution policies -• they need to work with some proprietary, custom datasource. E.g., a robotics company might have a datasource specific to ROS topics that are particular to that company -• they want to deploy their integration without going through the community review/release process<br /> - Forcing developers to fork the repo in order to add custom integrations is a deterrent to building an ecosystem.</p> - -<ol start="3"><li>Adding integrations is less complex, as developers don't need to worry about, e.g., conflicting versions of Guava required by the <code>BigQueryRelation</code> and the <code>HBaseRelation</code>.</li> -</ol> - -<p>The tradeoff here is that users need to declare the ~Marquez~OpenLineage integration dependencies and deploy them separately from the main jar. Statically typed dataset integrations should be loaded by the Spark <code>ClassLoader</code> (e.g., declared in the <code>spark.jars.packages</code> or <code>spark.jars</code> configuration) in order to link to the Datasource classes they depend on. In terms of developer experience, I don't think this pattern is foreign to Spark users today.</p> - -<p>Notably, nothing prohibits users from bundling their own superjars that contain all of their dependencies and deploying them to their Spark clusters. Similarly, nothing prevents integration authors from using reflection in their code. Users are given the freedom to extend and package their integrations in a way that suits them without enforcing an opaque, black-box model.</p> - -<p>~As a separate issue, we can consider changing the ~Marquez~OpenLineage agent to be loaded by the <code>spark.extraListeners</code> configuration. This would allow users to reference all of the ~Marquez~OpenLineage integration dependencies by specifying the listener and the node visitors in <code>spark.jars.packages</code> so that all of the classes are loaded by the Spark <code>ClassLoader</code>. That eliminates any confusion about which components need to be loaded by which <code>ClassLoader</code>. It also simplifies testing and could enable integration in serverless environments, such as AWS Glue, where users don't have access to modify the JVM arguments.~</p> - -<p>&gt; This has already been implemented</p> - -<p><strong>Packaging Spark Integrations</strong></p> - -<p>Spark integration packages need to create a file in <code>META-INF/services/marquez.spark.agent.lifecycle.plan.DatasetSource</code> which declares a list of classes that implement the <code>openlineage.spark.agent.lifecycle.plan.DatasetSource</code> interface. During startup, the main ~Marquez~OpenLineage agent uses the JVM <code>ServiceLoader</code> to find all service implementations on the classpath (we need to specifically check the Spark <code>URLClassloader</code> to find implementations loaded by Spark at runtime). Combining jars requires concatenating the contents of the <code>META-INF/services/marquez.spark.agent.lifecycle.plan.DatasetSource</code> files so that all implementations are listed.</p> - -<p><strong>Deploying Dependencies</strong></p> - -<p>~Currently, users deploy the Marquez integration by adding the <code>javaagent:&amp;lt;jar&amp;gt;=&lt;http://marquez_url/path/to/job&gt;</code> argument to their spark-submit args. That doesn't change with this proposal.~</p> - -<p>What does change is that users would be required to specify the integrations they require in their <code>spark.jars</code> or <code>spark.jars.packages</code> configuration. As an example, the airflow example in the README would look like this:</p> - -<pre><code>from uuid import uuid4 -import os -... -job_name = 'job_name' - - -properties = { - 'spark.extraListeners': 'openlineage.spark.agent.OpenLineageSparkListener', - 'spark.jar.packages': 'io.openlineage:openlineage_spark:0.0.1', - 'spark.openlineage.host': os.environ.get('MARQUEZ_URL'), - 'spark.openlineage.namespace': os.getenv('MARQUEZ_NAMESPACE', 'default'), - 'spark.openlineage.apiKey': os.environ.get('MARQUEZ_API_KEY'), - 'spark.openlineage.parentRunId': "{{task_run_id(run_id, task)}}", - 'spark.openlineage.parentJobName': '{{ task.task_id }}' -} - -t1 = DataProcPySparkOperator( - task_id=job_name, - gcp_conn_id='google_cloud_default', - project_id='project_id', - cluster_name='cluster-name', - region='us-west1', - main='&lt;gs://bucket/your-prog.py&gt;', - job_name=job_name, - dataproc_pyspark_properties=properties, - - &lt;b&gt;#Both&lt;/b&gt; bigquery relation and bigquery openlineage integration declared here - dataproc_pyspark_jars='&lt;gs://spark-lib/bigquery/spark-bigquery-latest_2.12.jar,gs://my_bucket/path/to/bigquery-integration.jar&gt;', - dag=dag) -</code></pre> - -<p>Alternatively, a cluster can be created by loading the correct jars into the Spark jars directory, as described in the <a href="https://github.com/GoogleCloudDataproc/initialization-actions/tree/master/connectors">BigQuery connector docs</a>. In the same way, the ~Marquez~OpenLineage integrations would need to be installed on the driver node. In either case, the person responsible for loading the bigquery jars on the classpath is also responsible for loading the Marquez integration. Dependencies can be bundled so that the Marquez integration and the BigQueryRelation class are all loaded in the same jar.</p> -
- -
-
Labels
- proposal -
- -
-
Comments
- 1 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 19:27:37
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/169|#169 Add CHANGELOG.md>
-
- <p>This PR adds the <code>CHANGELOG.md</code> for the initial public release of OpenLineage.</p> -
- -
-
Reviewers
- julienledem -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:05:28
 -
-

[OpenLineage/OpenLineage] Issue opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/170|#170 Airflow DAG serialization error on Clear>
-
- <p>When attempting to Clear a DAG on Google's CloudComposer, I get an error:</p> - -<pre><code>TypeError: can't pickle _thread.RLock objects - -</code></pre> - -<p>This happens with a simple example DAG, such as</p> - -<pre><code>from marquez_airflow import DAG -from airflow.contrib.operators.bigquery_operator import BigQueryOperator -from airflow.utils.dates import days_ago - -default_args = { - 'owner': 'datascience', - 'depends_on_past': False, - 'start_date': days_ago(1), - 'email_on_failure': False, - 'email_on_retry': False, - 'email': ['&lt;a href="mailto:datascience@example.com"&gt;datascience@example.com&lt;/a&gt;'] -} - -dag = DAG( - 'delivery_times_7_days', - schedule_interval='@hourly', - catchup=False, - default_args=default_args, - description='Determine weekly top delivery times by restaurant.' -) - -t1 = BigQueryOperator( - task_id='if_not_exists', - sql=''' - CREATE TABLE IF NOT EXISTS food_delivery.top_delivery_times ( - order_id INT64, - order_placed_on TIME, - order_dispatched_on TIME, - order_delivered_on TIME, - order_delivery_time INT64, - customer_email STRING, - restaurant_id INT64, - driver_id INT64 - ) - ''', - use_legacy_sql=False, - dag=dag -) - -t2 = BigQueryOperator( - task_id='insert', - sql=''' - SELECT order_id, - order_placed_on, - order_dispatched_on, - order_delivered_on, - EXTRACT(MINUTE FROM order_delivered_on) - EXTRACT(MINUTE FROM order_placed_on) AS order_delivery_time, - customer_email, - restaurant_id, - driver_id - FROM food_delivery.delivery_7_days - ORDER BY order_delivery_time DESC - LIMIT 1 - ''', - destination_dataset_table='bq-airflow-marquez.food_delivery.top_delivery_times', - use_legacy_sql=False, - dag=dag -) - -t1 &amp;gt;&amp;gt; t2 -</code></pre> - -<p>It should be noted that the issue happens with DAGs that don't include the BigQueryOperator, such as Spark DAGs.</p> -
- -
-
Comments
- 3 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:05:40
 -
-

[OpenLineage/OpenLineage] Issue opened by mstrbac

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/171|#171 Marquez doesn't collect dataset metadata for BigQueryOperator in Airflow>
-
- <p>Marquez doesn't collect dataset metadata for BigQueryOperator in Airflow. To collect BigQuery metadata, I tried the Airflow <a href="https://github.com/MarquezProject/marquez/tree/main/examples/airflow">example</a> from the repo, but instead of <code>PostgresOperator</code>, I used <code>BigQueryOperator</code>. So, for example, I modified operators in <code>counter.py</code> as:</p> - -<pre><code>from airflow.contrib.operators.bigquery_operator import BigQueryOperator - -t1 = BigQueryOperator( - task_id='if_not_exists', - bigquery_conn_id=BQ_CONN_ID, - sql=''' - CREATE TABLE IF NOT EXISTS `my-project.my_dataset.counts` ( - value INTEGER - );''', - use_legacy_sql=False, - dag=dag -) - -</code></pre> - -<pre><code>t2 = BigQueryOperator( - task_id='inc', - bigquery_conn_id=BQ_CONN_ID, - sql=''' - INSERT INTO `my-project.my_dataset.counts` (value) - VALUES ({{params.value}}) - ''', - params={ - "value": random.randint(1, 10) - }, - use_legacy_sql=False, - dag=dag -) - -</code></pre> - -<p>The dags ran successfully in Airflow, the counts and sums tables have been created on GCP, and in Marquez UI, metadata for jobs can be seen, but not for datasets.</p> -
- -
-
Comments
- 2 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:05:50
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/172|#172 Ensure Spark/airflow integrations follow the job naming convention>
-
- <p><a href="https://github.com/OpenLineage/OpenLineage/issues/66">#66</a></p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:06:04
 -
-

[OpenLineage/OpenLineage] Issue opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/173|#173 [INTEGRATION] Research tracking lineage scheduled bigquery queries>
-
- <p><a href="https://cloud.google.com/bigquery/docs/scheduling-queries">https://cloud.google.com/bigquery/docs/scheduling-queries</a></p> -
- -
-
Assignees
- mobuchowski -
- -
-
Comments
- 2 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:06:18
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/174|#174 Allow Marquez to be added to Airflow through Airflow configuration without having to import marquez_airflow.DAG>
-
- <p>This is now replaced by the OpenLineage API</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:06:31
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/175|#175 Add a facet reporting unknown inputs/outputs in marquez-spark>
-
- <p>Each leaf in the logical plan is an input and the root is the output.<br /> -When a Leaf is not understood we should track an "UnknownInput" facet with as much details as we can extract at runtime about what this input actually is.<br /> -The same applies to unknown actions/command for the output (root of the plan).</p> -
- -
-
Comments
- 1 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:06:39
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/176|#176 Add documentation on how to configure Marquez on astronomer>
-
- <p>In the marquez-airflow doc we should have a section on how to configure Marquez on Astronomer.</p> - -<p><a href="https://www.astronomer.io/docs/cloud/stable/deploy/configure-deployment#set-environment-variables">https://www.astronomer.io/docs/cloud/stable/deploy/configure-deployment#set-environment-variables</a></p> - -<p><a href="https://github.com/MarquezProject/marquez/tree/main/integrations/airflow#configuration">https://github.com/MarquezProject/marquez/tree/main/integrations/airflow#configuration</a></p> -
- -
-
Comments
- 1 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:06:50
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/177|#177 [INTEGRATION] Add Airflow SLA facet>
-
- <p>Airflow allow users to define a SLA for a task.<br /> -This should be added as an airflow custom facet when available</p> -
- -
-
Assignees
- mobuchowski -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:07:06
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/178|#178 [INTEGRATION][Airflow] Collect the query plan in Snowflake integration>
-
- <p>Given the Snowflake query id, the explain plan can be retrieved with the following query:<br /> -(using the query id <code>019bbdfa-0400-571a-0000-00301d038151</code> as an example here)</p> - -<pre><code>select system$explain_plan_json('019bbdfa-0400-571a-0000-00301d038151') as explain_plan; -</code></pre> - -<p>This returns a json representation of the plan that should go in a Snowflake facet</p> - -<p>Here is an example of a plan:</p> - -<pre><code>{ - "GlobalStats":{ - "partitionsTotal":4, - "partitionsAssigned":4, - "bytesAssigned":2042880 - }, - "Operations":[ - [ - { - "id":0, - "operation":"Result", - "expressions":[ - "number of rows inserted" - ] - }, - { - "id":1, - "parent":0, - "operation":"Insert", - "objects":[ - "DEMO_DB.PUBLIC.FOO" - ] - }, - { - "id":2, - "parent":1, - "operation":"SortWithLimit", - "expressions":[ - "sortKey: [SUM(LINEITEM.L_EXTENDEDPRICE ** (1 - LINEITEM.L_DISCOUNT)) DESC NULLS FIRST]", - "rowCount: 20" - ] - }, - { - "id":3, - "parent":2, - "operation":"InnerJoin", - "expressions":[ - "joinKey: (ORDERS.O_ORDERKEY = LINEITEM.L_ORDERKEY)" - ] - }, - { - "id":4, - "parent":3, - "operation":"InnerJoin", - "expressions":[ - "joinKey: (NATION.N_NATIONKEY = CUSTOMER.C_NATIONKEY)" - ] - }, - { - "id":5, - "parent":4, - "operation":"TableScan", - "objects":[ - "SNOWFLAKE_SAMPLE_DATA.TPCH_SF001.NATION" - ], - "expressions":[ - "N_NATIONKEY", - "N_NAME" - ], - "partitionsAssigned":1, - "partitionsTotal":1, - "bytesAssigned":2048 - }, - { - "id":6, - "parent":4, - "operation":"InnerJoin", - "expressions":[ - "joinKey: (ORDERS.O_CUSTKEY = CUSTOMER.C_CUSTKEY)" - ] - }, - { - "id":7, - "parent":6, - "operation":"Filter", - "expressions":[ - "(ORDERS.O_ORDERDATE &amp;gt;= '1993-10-01') AND (ORDERS.O_ORDERDATE &amp;lt; '1993-11-01')" - ] - }, - { - "id":8, - "parent":7, - "operation":"TableScan", - "objects":[ - "SNOWFLAKE_SAMPLE_DATA.TPCH_SF001.ORDERS" - ], - "expressions":[ - "O_ORDERKEY", - "O_CUSTKEY", - "O_ORDERDATE" - ], - "partitionsAssigned":1, - "partitionsTotal":1, - "bytesAssigned":407552 - }, - { - "id":9, - "parent":6, - "operation":"JoinFilter", - "expressions":[ - "joinKey: (NATION.N_NATIONKEY = CUSTOMER.C_NATIONKEY)" - ] - }, - { - "id":10, - "parent":9, - "operation":"TableScan", - "objects":[ - "SNOWFLAKE_SAMPLE_DATA.TPCH_SF001.CUSTOMER" - ], - "expressions":[ - "C_CUSTKEY", - "C_NAME", - "C_ADDRESS", - "C_NATIONKEY", - "C_PHONE", - "C_ACCTBAL", - "C_COMMENT" - ], - "partitionsAssigned":1, - "partitionsTotal":1, - "bytesAssigned":109568 - }, - { - "id":11, - "parent":3, - "operation":"Filter", - "expressions":[ - "LINEITEM.L_RETURNFLAG = 'R'" - ] - }, - { - "id":12, - "parent":11, - "operation":"JoinFilter", - "expressions":[ - "joinKey: (ORDERS.O_ORDERKEY = LINEITEM.L_ORDERKEY)" - ] - }, - { - "id":13, - "parent":12, - "operation":"TableScan", - "objects":[ - "SNOWFLAKE_SAMPLE_DATA.TPCH_SF001.LINEITEM" - ], - "expressions":[ - "L_ORDERKEY", - "L_EXTENDEDPRICE", - "L_DISCOUNT", - "L_RETURNFLAG" - ], - "partitionsAssigned":1, - "partitionsTotal":1, - "bytesAssigned":1523712 - } - ] - ] -} -</code></pre> - -<p>For the following query:</p> - -<pre><code> insert into DEMO_DB.PUBLIC.foo (c_custkey, c_name, revenue, c_acctbal, - n_name, - c_address, - c_phone, - c_comment) SELECT - c_custkey, - c_name, - sum(l_extendedprice ** (1 - l_discount)) as revenue, - c_acctbal, - n_name, - c_address, - c_phone, - c_comment -FROM - customer, - orders, - lineitem, - nation -WHERE - c_custkey = o_custkey - AND l_orderkey = o_orderkey - AND o_orderdate &amp;gt;= date '1993-10-01' - AND o_orderdate &amp;lt; date '1993-11-01' - AND l_returnflag = 'R' - AND c_nationkey = n_nationkey -GROUP BY - c_custkey, - c_name, - c_acctbal, - c_phone, - n_name, - c_address, - c_comment -ORDER BY - revenue desc -LIMIT 20; -</code></pre> - -<p>note that the nodes <code>"operation":"TableScan"</code> have the input table in their <code>objects</code> field.<br /> -and in this case the node <code>"operation":"Insert"</code> has the output in the <code>objects</code> filed.</p> -
- -
-
Assignees
- mobuchowski -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:07:17
 -
-

[OpenLineage/OpenLineage] Issue opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/179|#179 [INTEGRATION] [Airflow] explore hooking via airflow plugins>
-
- <p>Right now, we hook to airflow by subclassing DAG and asking users to use our subclass instead of <code>airflow.models.DAG</code>.<br /> -Explore hooking our changes to DAG via plugin mechanism: <a href="https://airflow.apache.org/docs/apache-airflow/stable/plugins.html">https://airflow.apache.org/docs/apache-airflow/stable/plugins.html</a></p> -
- -
-
Comments
- 3 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:07:26
 -
-

[OpenLineage/OpenLineage] Issue opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/180|#180 [INTEGRATION] [Spark] double triggers on DataFrame -&gt; RDD conversion>
-
- <p>Discovered when running tests, when a DataFrame is converted to an RDD (such as when writing to a JDBC relation), the marquez listener is double-triggered- once for the DataFrame operation and again for the RDD operation. Fortunately, the original DataFrame LogicalPlan contains the entire input/output graph, so no information is lost. However, the second set of events does post a new job with no inputs or outputs to the Marquez endpoint.</p> - -<p>Example of two posted events- one from the DataFrame and one from the RDD:</p> - -<pre><code>Publishing sql event LineageEvent(eventType=COMPLETE, eventTime=2021-01-01T00:00Z, run=LineageEvent.Run(runId=39f36ea5-bd33-4ff5-9a64-00fa2f439d21, facets=LineageEvent.RunFacet(nominalTime=null, parent=LineageEvent.ParentRunFacet(run=LineageEvent.RunLink(runId=ab478a6e-0226-46bb-b5c3-2b23179a1277), job=LineageEvent.JobLink(namespace=Namespace, name=ParentJob)), additional={spark.logicalPlan=LogicalPlanFacet(plan=[{"class":"org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand","num_children":0,"query":[{"class":"org.apache.spark.sql.catalyst.plans.logical.Filter","num-children":1,"condition":[{"class":"org.apache.spark.sql.catalyst.expressions.GreaterThan","num_children":2,"left":0,"right":1},{"class":"org.apache.spark.sql.catalyst.expressions.AttributeReference","num_children":0,"name":"age","dataType":"long","nullable":true,"metadata":{},"exprId":{"product_class":"org.apache.spark.sql.catalyst.expressions.ExprId","id":6,"jvmId":"2bb20251-cb0e-4586-a2b0-ae90290f14e4"},"qualifier":[]},{"class":"org.apache.spark.sql.catalyst.expressions.Cast","num-children":1,"child":0,"dataType":"long","timeZoneId":"America/Los_Angeles"},{"class":"org.apache.spark.sql.catalyst.expressions.Literal","num_children":0,"value":"16","dataType":"integer"}],"child":0},{"class":"org.apache.spark.sql.execution.datasources.LogicalRelation","num-children":0,"relation":null,"output":[[{"class":"org.apache.spark.sql.catalyst.expressions.AttributeReference","num-children":0,"name":"age","dataType":"long","nullable":true,"metadata":{},"exprId":{"product-class":"org.apache.spark.sql.catalyst.expressions.ExprId","id":6,"jvmId":"2bb20251_cb0e_4586_a2b0_ae90290f14e4"},"qualifier":[]}],[{"class":"org.apache.spark.sql.catalyst.expressions.AttributeReference","num_children":0,"name":"name","dataType":"string","nullable":true,"metadata":{},"exprId":{"product_class":"org.apache.spark.sql.catalyst.expressions.ExprId","id":7,"jvmId":"2bb20251-cb0e-4586-a2b0-ae90290f14e4"},"qualifier":[]}]],"isStreaming":false}],"dataSource":null,"options":null,"mode":null}])})), job=LineageEvent.Job(namespace=Namespace, name=word_count.execute_save_into_data_source_command, facets=null), inputs=[LineageEvent.Dataset(namespace=sqlite:file:///var/folders/x_/47q8z0xx2357hrl3w6k9xf_80000gn/T/junit12164537964587498492/sqlite/database, name=data_table, facets=LineageEvent.DatasetFacet(documentation=null, schema=LineageEvent.SchemaDatasetFacet(fields=[LineageEvent.SchemaField(name=age, type=decimal(20,0), description=null), LineageEvent.SchemaField(name=name, type=string, description=null)]), dataSource=LineageEvent.DatasourceDatasetFacet(name=sqlite:file:///var/folders/x_/47q8z0xx2357hrl3w6k9xf_80000gn/T/junit12164537964587498492/sqlite/database, uri=sqlite:file:///var/folders/x_/47q8z0xx2357hrl3w6k9xf_80000gn/T/junit12164537964587498492/sqlite/database), description=null, additional={}))], outputs=[LineageEvent.Dataset(namespace=sqlite:file:///var/folders/x_/47q8z0xx2357hrl3w6k9xf_80000gn/T/junit12164537964587498492/sqlite/database, name=data_table, facets=LineageEvent.DatasetFacet(documentation=null, schema=LineageEvent.SchemaDatasetFacet(fields=[LineageEvent.SchemaField(name=age, type=decimal(20,0), description=null), LineageEvent.SchemaField(name=name, type=string, description=null)]), dataSource=LineageEvent.DatasourceDatasetFacet(name=sqlite:file:///var/folders/x_/47q8z0xx2357hrl3w6k9xf_80000gn/T/junit12164537964587498492/sqlite/database, uri=sqlite:file:///var/folders/x_/47q8z0xx2357hrl3w6k9xf_80000gn/T/junit12164537964587498492/sqlite/database), description=null, additional={}))], producer=&lt;https://github.com/MarquezProject/marquez/tree/0.12.0/integrations/spark)&gt; -Publishing RDD event LineageEvent(eventType=COMPLETE, eventTime=2021-01-01T00:00Z, run=LineageEvent.Run(runId=ab478a6e-0226-46bb-b5c3-2b23179a1277, facets=LineageEvent.RunFacet(nominalTime=null, parent=LineageEvent.ParentRunFacet(run=LineageEvent.RunLink(runId=ab478a6e-0226-46bb-b5c3-2b23179a1277), job=LineageEvent.JobLink(namespace=Namespace, name=ParentJob)), additional={})), job=LineageEvent.Job(namespace=Namespace, name=word_count, facets=null), inputs=[], outputs=[], producer=&lt;https://github.com/MarquezProject/marquez/tree/0.12.0/integrations/spark)&gt; - -</code></pre> - -<p>Note that the RDD event has no inputs/outputs and has a distinct run id.</p> -
- -
-
Assignees
- OleksandrDvornik -
- -
-
Comments
- 1 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:07:40
 -
-

[OpenLineage/OpenLineage] Issue opened by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/181|#181 [INTEGRATION] [Spark] Making the spark integration extensible>
-
- <p><strong>Context</strong></p> - -<p>The Spark integration relies on instrumentation achieved by adding a jar to the <code>javaagent</code> argument on the command line when starting a Spark job. When the JVM starts up, the application <code>ClassLoader</code> loads both the instrumentation jar as well as the Spark, Hadoop, and other dependency jars that are on the classpath. When the <code>SparkContext</code> is loaded, Spark parses its own configuration and finds additional jars that have been specified, either on the command line or in a configuration map that can be populated at runtime. The jars are (sometimes) downloaded and added to a <code>MutableURLClassLoader</code> that is owned and managed by the <code>SparkContext</code>.</p> - -<p>Since the Marquez integration is loaded by the top level application <code>ClassLoader</code>, and Java’s <code>ClassLoaders</code> are ignorant of their child <code>ClassLoaders</code>, the integration is only able to find classes that are loaded by that top level <code>ClassLoader</code>. BigQuery and other dependencies may be specified in the <code>spark.jars</code> configuration, so that they are loaded by that child <code>MutableURLClassLoader</code>, hence invisible to the Marquez agent classes. While it is possible from the integration to detect that BigQuery classes are available, attempting to instantiate the classes that actually deal with those classes cause <code>NoClassDefFoundErrors</code>, since the <code>ClassLoader</code> can’t find the classes loaded by the child loader.</p> - -<p>The current workaround relies on <a href="https://github.com/MarquezProject/marquez/pull/1180">instrumentation</a> to invoke methods from the parent <code>ClassLoader</code> without needing to directly access the <code>BigQueryRelation</code> class or related classes.</p> - -<p>I think the instrumentation approach has the following drawbacks</p> - -<ol><li>We lose type safety and the ability to easily unit test the instrumentation code</li><li>Failure to instrument the code at runtime leaves users blind to the missing dataset nodes</li><li>Isn't extensible to support other integrations</li> -</ol> - -<p>To be fair, it has the following advantages</p> - -<ol><li>Only a single jar is needed at runtime</li><li>It's possible for a single jar to support multiple, statically non-compatible versions of the BigQuery integration using conditional checks (e.g., <code>if ( class.hasMethod("X")) generateXSpecificCode()</code>)</li> -</ol> - -<p>Reflection has been suggested as an alternative strategy, but I think the pros/cons are largely the same as the instrumentation approach- it uses indirect method invocation to circumvent static type checking.</p> - -<p><strong>Proposal</strong></p> - -<p>Instead of instrumentation or reflection, I propose a more extensible method of loading and discovering node visitors. The basic strategy is</p> - -<ol><li>Create a common interface for visiting logical plan nodes- something that extends the <code>PartitalFunction&amp;lt;LogicalPlan, List&amp;lt;Dataset&amp;gt;&amp;gt;</code> interface we already use</li><li>Take advantage of the JVM's <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html">ServiceLoader</a> capabilities to enable implementations on the classpath to be found at runtime</li><li>Package node visitors in different subprojects that generate their own jars</li><li>Entrust Spark users to declare dependencies on the node visitors they need using the <code>spark.jars.packages</code> or <code>spark.jars</code> configuration properties</li> -</ol> - -<p>The main advantages I see here are</p> - -<ol><li>Static typing, easy to read code, and easy unit testing</li><li>Extensible framework to support new integrations<br /> -This point, I feel, is extremely important. While the single jar deployment is convenient, it creates an unnecessary barrier for users who want to take part in the OpenLineage ecosystem, but can't/won't contribute directly to Marquez or OpenLineage. This may be because</li> -</ol> - -<p>• their company has strict open source contribution policies -• they need to work with some proprietary, custom datasource. E.g., a robotics company might have a datasource specific to ROS topics that are particular to that company -• they want to deploy their integration without going through the community review/release process<br /> - Forcing developers to fork the repo in order to add custom integrations is a deterrent to building an ecosystem.</p> - -<ol start="3"><li>Adding integrations is less complex, as developers don't need to worry about, e.g., conflicting versions of Guava required by the <code>BigQueryRelation</code> and the <code>HBaseRelation</code>.</li> -</ol> - -<p>The tradeoff here is that users need to declare the Marquez integration dependencies and deploy them separately from the main agent jar. Statically typed dataset integrations should be loaded by the Spark <code>ClassLoader</code> (e.g., declared in the <code>spark.jars.packages</code> or <code>spark.jars</code> configuration) in order to link to the Datasource classes they depend on. In terms of developer experience, I don't think this pattern is foreign to Spark users today.</p> - -<p>Notably, nothing prohibits users from bundling their own superjars that contain all of their dependencies and deploying them to their Spark clusters. Similarly, nothing prevents integration authors from using reflection in their code. Users are given the freedom to extend and package their integrations in a way that suits them without enforcing an opaque, black-box model.</p> - -<p>As a separate issue, we can consider changing the Marquez agent to be loaded by the <code>spark.extraListeners</code> configuration. This would allow users to reference all of the Marquez integration dependencies by specifying the listener and the node visitors in <code>spark.jars.packages</code> so that all of the classes are loaded by the Spark <code>ClassLoader</code>. That eliminates any confusion about which components need to be loaded by which <code>ClassLoader</code>. It also simplifies testing and could enable integration in serverless environments, such as AWS Glue, where users don't have access to modify the JVM arguments.</p> - -<p><strong>Packaging Spark Integrations</strong></p> - -<p>Spark integration packages need to create a file in <code>META-INF/services/marquez.spark.agent.lifecycle.plan.DatasetSource</code> which declares a list of classes that implement the <code>marquez.spark.agent.lifecycle.plan.DatasetSource</code> interface. During startup, the main Marquez agent uses the JVM <code>ServiceLoader</code> to find all service implementations on the classpath (we need to specifically check the Spark <code>URLClassloader</code> to find implementations loaded by Spark at runtime). Combining jars requires concatenating the contents of the <code>META-INF/services/marquez.spark.agent.lifecycle.plan.DatasetSource</code> files so that all implementations are listed.</p> - -<p><strong>Deploying Dependencies</strong></p> - -<p>Currently, users deploy the Marquez integration by adding the <code>javaagent:&amp;lt;jar&amp;gt;=&lt;http://marquez_url/path/to/job&gt;</code> argument to their spark-submit args. That doesn't change with this proposal.</p> - -<p>What does change is that users would be required to specify the integrations they require in their <code>spark.jars</code> or <code>spark.jars.packages</code> configuration. As an example, the airflow example in the README would look like this:</p> - -<pre><code>from uuid import uuid4 -import os -... -job_name = 'job_name' - -jar = 'marquez-spark-0.13.1.jar' -files = [f"&lt;https://repo1.maven.org/maven2/io/github/marquezproject/marquez-spark/0.13.1/marquez-spark-0.13.1.jar&gt;"] -properties = { - 'spark.driver.extraJavaOptions': - f"-javaagent:{jar}={os.environ.get('MARQUEZ_URL')}/api/v1/namespaces/{os.getenv('MARQUEZ_NAMESPACE', 'default')}/jobs/{job_name}/runs/{uuid4()}?api_key={os.environ.get('MARQUEZ_API_KEY')}" -} - -t1 = DataProcPySparkOperator( - task_id=job_name, - gcp_conn_id='google_cloud_default', - project_id='project_id', - cluster_name='cluster-name', - region='us-west1', - main='&lt;gs://bucket/your-prog.py&gt;', - job_name=job_name, - dataproc_pyspark_properties=properties, - files=files, - &lt;b&gt;#Both&lt;/b&gt; bigquery relation and bigquery marquez integration declared here - dataproc_pyspark_jars='&lt;gs://spark-lib/bigquery/spark-bigquery-latest_2.12.jar,gs://my_bucket/path/to/bigquery-integration.jar&gt;', - dag=dag) -</code></pre> - -<p>Alternatively, a cluster can be created by loading the correct jars into the Spark jars directory, as described in the <a href="https://github.com/GoogleCloudDataproc/initialization-actions/tree/master/connectors">BigQuery connector docs</a>. In the same way, the…</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:07:49
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/182|#182 [INTEGRATION] [Airflow] Add support for the S3ToRedshiftOperator and RedshiftToS3Operator to the marquez-airflow integration>
-
- <p><strong>Goal:</strong></p> - -<p>Add lineage coverage for the redshift &lt;-&gt; s3 operators<br /> -<a href="https://github.com/apache/airflow/blob/v1-10-stable/airflow/operators/redshift_to_s3_operator.py">https://github.com/apache/airflow/blob/v1-10-stable/airflow/operators/redshift<em>to</em>s3<em>operator.py</a><br /> -<a href="https://github.com/apache/airflow/blob/v1-10-stable/airflow/operators/s3_to_redshift_operator.py">https://github.com/apache/airflow/blob/v1-10-stable/airflow/operators/s3</em>to<em>redshift</em>operator.py</a></p> - -<p>Please reffer to the OpenLineage naming spec:<br /> -<a href="https://github.com/OpenLineage/OpenLineage/pull/31/files">https://github.com/OpenLineage/OpenLineage/pull/31/files</a></p> -
- -
-
Assignees
- mobuchowski -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:08:01
 -
-

[OpenLineage/OpenLineage] Issue opened by nizardeen

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/183|#183 [INTEGRATION] [Airflow] Support for S3 Operators in marquez_airflow>
-
- <p>Need support for S3 operator and extractor in marquez_airflow to run a DAG.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:08:13
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/184|#184 package level job metadata>
-
- <p><strong>Purpose</strong></p> - -<p>To expose information like the version of the job and its source repository, Marquez needs a mechanism to provide metadata in the job package when deploying to a scheduler.<br /> -An example of this is deploying DAGs to an Airflow instance, but this is applicable to all package deployments in general.<br /> -In some instances workflows/jobs are directly pulled from a source control system from the scheduler allowing some metadata to be inferred from the environment. In other instances, the jobs are packaged and deployed to a scheduler (example: google cloud composer or astronomer). This doc concerns the later case of providing metadata related to the source code of the job at build time.</p> - -<p><strong>Proposal</strong></p> - -<p>We define a metadata file to be added at the root of the package/deployment location. This file is named <strong>.OpenLineage.job.facets.json</strong> and is a json file with the following initial properties (that can be expanded later).</p> - -<p><strong>Properties:</strong></p> - -<p>• <strong>sourceCodeLocation.type</strong>: the source control system<br /> - Examples: GIT, SVN, ... -• <strong>sourceCodeLocation.url</strong>: the URL to the repository<br /> - Examples: - • GIT: <a href="mailto:git@github.com">git@github.com</a>:{org}/{repo}.git or <a href="https://github.com/%7Borg%7D/%7Brepo%7D.git|https://github.com/{org}/{repo}.git">https://github.com/%7Borg%7D/%7Brepo%7D.git|https://github.com/{org}/{repo}.git</a> - • SVN: svn://&lt;your<em>ip&gt;/&lt;repository</em>name&gt; -• <strong>sourceCodeLocation.path</strong>: the path in the repo containing the source files<br /> - Example: path/to/my/dags<br /> - It is intended that we can reconstruct a URL from that path pointing to the file defining a job. <a href="https://github.com/%7Borg%7D/%7Brepo%7D/tree/%7Bverion%7D/%7Bpath%7D/%7Bfile|https://github.com/{org}/{repo}/tree/{verion}/{path}/{file">https://github.com/%7Borg%7D/%7Brepo%7D/tree/%7Bverion%7D/%7Bpath%7D/%7Bfile|https://github.com/{org}/{repo}/tree/{verion}/{path}/{file</a> path in the package} -• <strong>sourceCodeLocation.version</strong>: the current version deployed (not a branch name, the actual unique version)<br /> - Examples: - • Git: the git sha - • Svn: the revision number -• <strong>sourceCodeLocation.tag</strong>: optional tag name<br /> - Examples: {tag name} -• <strong>sourceCodeLocation.branch</strong>:optional branch name</p> - -<p><strong>JSON Schema</strong></p> - -<pre><code>{ - "$schema": "&lt;https://json-schema.org/draft/2019-09/schema&gt;", - "type": "object", - "properties": { - "sourceCodeLocation": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "url": { - "type": "string" - }, - "path": { - "type": "string" - }, - "version": { - "type": "string" - }, - "branch": { - "type": "string" - } - }, - "required": [ - "type", - "url", - "path", - "version", - "branch" - ] - } - }, - "required": [ - "sourceCodeLocation" - ] -} - -</code></pre> - -<p><strong>Example:</strong></p> - -<p>For the DAG defined at: <a href="https://github.com/MarquezProject/marquez-airflow-quickstart/blob/693e35482bc2e526ced2b5f9f76ef83dec6ec691/dags/dummy_example.py">https://github.com/MarquezProject/marquez-airflow-quickstart/blob/693e35482bc2e526ced2b5f9f76ef83dec6ec691/dags/dummy_example.py</a></p> - -<pre><code>{ - "sourceCodeLocation": { - "type": "GIT", - "url: "&lt;https://github.com/MarquezProject/marquez-airflow-quickstart.git&gt;", - "path": "dags", - "version": "693e35482bc2e526ced2b5f9f76ef83dec6ec691", - "branch": "master" - } -} - -</code></pre> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 20:08:22
 -
-

[OpenLineage/OpenLineage] Issue opened by pomdtr

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/185|#185 Handling of Dataset Partitions>
-
- <p>If i have a job scheduled to push a partition to a table each hour.<br /> -And a job running daily consuming the 24 last partitions created by the previous job.<br /> -Would marquez be able to modelize this dependency ?</p> - -<p>I don't see a way to use multiple datasetVersion as input in a request.</p> -
- -
-
Comments
- 2 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-11 21:00:24
 -
-

[OpenLineage/OpenLineage] Pull request opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/186|#186 Create integration.md>
-
- <p>This adds an issue template for integrations</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 06:20:57
 -
-

[OpenLineage/OpenLineage] Issue opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/187|#187 [PROPOSAL] [SPARK] Add support of HiveTableRelation >
-
- <p><strong>Purpose:</strong><br /> -We should support HiveTableRelation in spark integration.<br /> -if we call ( <code>spark.sql("create table lineage_test.src_table as select ** from mytempTable")</code>) we will get event with empty datasets. Please take a look in Event sample. But all required data exist in LogicalPlan</p> - -<p><strong>Proposed implementation</strong><br /> -Add Visitor for HiveTableRelation and get required data</p> - -<p><strong>Event sample</strong></p> - -<p>``` -{ - "eventType":"START", - "eventTime":"2021<em>08</em>11T10:28:19.247Z", - "run":{ - "runId":"cea98e8e-980b-4cdd-83a2-ccdfe5b1fe36", - "facets":{ - "parent":{ - "<em>producer":"<a href="https://github.com/OpenLineage/OpenLineage/tree/0.1-SNAPSHOT/integration/spark">https://github.com/OpenLineage/OpenLineage/tree/0.1-SNAPSHOT/integration/spark</a>", - "</em>schemaURL":"<a href="https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json#ParentRunFacet">https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json#ParentRunFacet</a>", - "run":{ - "runId":"" - }, - "job":{ - "namespace":"spark<em>integration", - "name":"default" - } - }, - "spark.logicalPlan":{ - "</em>producer":"<a href="https://github.com/OpenLineage/OpenLineage/tree/0.1-SNAPSHOT/integration/spark">https://github.com/OpenLineage/OpenLineage/tree/0.1-SNAPSHOT/integration/spark</a>", - "<em>schemaURL":"<a href="https://github.com/OpenLineage/OpenLineage/tree/0.1-SNAPSHOT/integration/spark/facets/spark-2.4/v1/logicalPlanFacet">https://github.com/OpenLineage/OpenLineage/tree/0.1-SNAPSHOT/integration/spark/facets/spark-2.4/v1/logicalPlanFacet</a>", - "plan":[ - { - "class":"org.apache.spark.sql.hive.execution.CreateHiveTableAsSelectCommand", - "num-children":1, - "tableDesc":{ - "product-class":"org.apache.spark.sql.catalyst.catalog.CatalogTable", - "identifier":{ - "product-class":"org.apache.spark.sql.catalyst.TableIdentifier", - "table":"dst</em>table", - "database":"lineage_test" - }, - "tableType":{ - "product-class":"org.apache.spark.sql.catalyst.catalog.CatalogTableType", - "name":"MANAGED" - }, - "storage":{ - "product-class":"org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat", - "inputFormat":"org.apache.hadoop.mapred.TextInputFormat", - "outputFormat":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "serde":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", - "compressed":false, - "properties":null - }, - "schema":{ - "type":"struct", - "fields":[</p> - -<pre><code> ] - }, - "provider":"hive", - "partitionColumnNames":[ - - ], - "owner":"", - "createTime":1628677698949, - "lastAccessTime":-1, - "createVersion":"", - "properties":null, - "unsupportedFeatures":[ - - ], - "tracksPartitionsInCatalog":false, - "schemaPreservesCase":true, - "ignoredProperties":null - }, - "query":0, - "outputColumnNames":"[firstname, middlename, lastname, id, gender, salary]", - "mode":null - }, - { - "class":"org.apache.spark.sql.catalyst.catalog.HiveTableRelation", - "num-children":0, - "tableMeta":{ - "product-class":"org.apache.spark.sql.catalyst.catalog.CatalogTable", - "identifier":{ - "product-class":"org.apache.spark.sql.catalyst.TableIdentifier", - "table":"src_table", - "database":"lineage_test" - }, - "tableType":{ - "product-class":"org.apache.spark.sql.catalyst.catalog.CatalogTableType", - "name":"MANAGED" - }, - "storage":{ - "product-class":"org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat", - "locationUri":null, - "inputFormat":"org.apache.hadoop.mapred.TextInputFormat", - "outputFormat":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "serde":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", - "compressed":false, - "properties":null - }, - "schema":{ - "type":"struct", - "fields":[ - { - "name":"firstname", - "type":"string", - "nullable":true, - "metadata":{ - - } - }, - { - "name":"middlename", - "type":"string", - "nullable":true, - "metadata":{ - - } - }, - { - "name":"lastname", - "type":"string", - "nullable":true, - "metadata":{ - - } - }, - { - "name":"id", - "type":"string", - "nullable":true, - "metadata":{ - - } - }, - { - "name":"gender", - "type":"string", - "nullable":true, - "metadata":{ - - } - }, - { - "name":"salary", - "type":"integer", - "nullable":true, - "metadata":{ - - } - } - ] - }, - "provider":"hive", - "partitionColumnNames":[ - - ], - "owner":"jovyan", - "createTime":1628677564000, - "lastAccessTime":0, - "createVersion":"3.1.2", - "properties":null, - "stats":null, - "unsupportedFeatures":[ - - ], - "tracksPartitionsInCatalog":true, - "schemaPreservesCase":true, - "ignoredProperties":null - }, - "dataCols":[ - [ - { - "class":"org.apache.spark.sql.catalyst.expressions.AttributeReference", - "num-children":0, - "name":"firstname", - "dataType":"string", - "nullable":true, - "metadata":{ - - }, - "exprId":{ - "product-class":"org.apache.spark.sql.catalyst.expressions.ExprId", - "id":107, - "jvmId":"44aa7950-ade6-4eaf-8ae0-2daaf30e5d00" - }, - "qualifier":[ - - ] - } - ], - [ - { - "cla… -</code></pre> -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 12:16:02
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/161|#161 Add lfaidata project badge and status>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 12:58:26
 -
-

[OpenLineage/OpenLineage] Issue closed by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/168|#168 [PROPOSAL] Making the spark integration extensible>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 15:46:56
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/188|#188 Add RELEASING.md and fix new-version.sh>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 15:52:41
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/169|#169 Add CHANGELOG.md>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 15:58:24
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/188|#188 Add RELEASING.md and fix new-version.sh>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 16:11:24
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/189|#189 Fix versioning for dbt>
-
- -
- -
-
Comments
- 1 -
- -
-
Reviewers
- julienledem, collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 16:48:50
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/189|#189 Fix versioning for dbt>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 19:09:38
 -
-

[OpenLineage/OpenLineage] Pull request merged by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/186|#186 Create integration.md>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 19:18:17
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/190|#190 Fix version config for py modules and ensure all setup.py are the same >
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 19:21:21
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/190|#190 Fix version config for py modules and ensure all setup.py are the same >
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 19:37:04
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/191|#191 Fix syntax in common setup.py>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 19:45:14
 -
-

[OpenLineage/OpenLineage] Pull request merged by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/191|#191 Fix syntax in common setup.py>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 19:51:55
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/192|#192 Install py client in build-integration-airflow CI job>
-
- -
- -
-
Comments
- 1 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 20:00:57
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/193|#193 [INTEGRATION] [dbt] job names should include the target and skip the "model" prefix. the namespace should be user provided>
-
- <p><strong>Integration target:</strong><br /> -We need to improve the job naming for the dbt integration.<br /> -<a href="https://github.com/OpenLineage/OpenLineage/tree/main/integration/dbt">https://github.com/OpenLineage/OpenLineage/tree/main/integration/dbt</a></p> - -<ol><li>In the dbt integration, we always have "model." as a prefix for job names. This is unnecessary as they all start with model.</li><li>Additionally a dbt project can be run with multiple targets defined in profiles:<br /> -<a href="https://docs.getdbt.com/dbt-cli/configure-your-profile#understanding-profiles">https://docs.getdbt.com/dbt-cli/configure-your-profile#understanding-profiles</a><br /> -Depending on the "target" we should have a different job name as they run on different dataset inputs and outputs.</li> -</ol> - -<p>Possibly this can be gathered form the <code>database</code> and <code>schema</code> fields in the node.</p> - -<ol start="3"><li>The NAMESPACE should be user provided.</li> -</ol> - -<p>the openlineage-dbt config should take a namespace like the openlineage-airflow config<br /> -<a href="https://github.com/OpenLineage/OpenLineage/tree/main/integration/airflow#http-backend-environment-variables">https://github.com/OpenLineage/OpenLineage/tree/main/integration/airflow#http-backend-environment-variables</a></p> -
- -
-
Assignees
- mobuchowski -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 20:13:54
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/194|#194 [INTEGRATION] [dbt] we should capture the result of "dbt test" runs>
-
- <p><strong>Integration target:</strong><br /> -dbt allows defining data quality checks:<br /> -<a href="https://docs.getdbt.com/docs/building-a-dbt-project/tests">https://docs.getdbt.com/docs/building-a-dbt-project/tests</a></p> - -<p>We should capture them in the same way we capture Great Expectations Data quality checks:<br /> -<a href="https://github.com/OpenLineage/OpenLineage/blob/main/integration/airflow/openlineage/airflow/extractors/ge-assertions-dataset-facet.json">https://github.com/OpenLineage/OpenLineage/blob/main/integration/airflow/openlineage/airflow/extractors/ge-assertions-dataset-facet.json</a></p> - -<p>(this probably means promoting this as a standard facet)</p> -
- -
-
Assignees
- mobuchowski -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 20:15:26
 -
-

[OpenLineage/OpenLineage] Pull request merged by collado-mike

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/192|#192 Install py client in build-integration-airflow CI job>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 20:32:07
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/195|#195 Fix release tag regex in CI>
-
- -
- -
-
Reviewers
- collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 20:40:53
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/195|#195 Fix release tag regex in CI>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 20:59:34
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/196|#196 Add source distribution to build-client-python CI job>
-
- -
- -
-
Comments
- 1 -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-12 21:16:28
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/196|#196 Add source distribution to build-client-python CI job>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-13 13:46:20
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/198|#198 Append `-SNAPSHOT` to `X.Y.Z-rc.*`>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-13 13:46:31
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/198|#198 Append `-SNAPSHOT` to `X.Y.Z-rc.*`>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-13 13:51:46
 -
-

[OpenLineage/OpenLineage] Pull request closed by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/199|#199 [1 of 2] Add sdist.sh>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-13 13:53:48
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/200|#200 [1 of 2] Fix python sdist>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-13 14:36:22
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/200|#200 [1 of 2] Fix python sdist>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-13 14:46:37
 -
-

[OpenLineage/OpenLineage] Pull request ready for review by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/197|#197 [2 of 2] Cleanup .circleci/config.yml>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-13 15:40:16
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/197|#197 [2 of 2] Cleanup .circleci/config.yml>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-13 18:43:03
 -
-

[OpenLineage/OpenLineage] New release Release - OpenLineage 0.1.0 published by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/releases/tag/0.1.0|Release - OpenLineage 0.1.0>
-
- <p>OpenLineage is an <em>Open Standard</em> for lineage metadata collection designed to record metadata for a job in execution. The initial public release includes:</p> - -<p>• <strong>An inital specification.</strong> The the inital version <a href="https://github.com/OpenLineage/OpenLineage/blob/0.1.0/spec/OpenLineage.md"><code>1-0-0</code></a> of the OpenLineage specification defines the core model and facets. -• <strong>Integrations</strong> that collect lineage metadata as OpenLineage events: - • <a href="https://github.com/OpenLineage/OpenLineage/tree/main/integration/airflow"><code>Apache Airflow</code></a> with support for BigQuery, Great Expectations, Postgres, Redshift, Snowflake - • <a href="https://github.com/OpenLineage/OpenLineage/tree/main/integration/spark"><code>Apache Spark</code></a> - • <a href="https://github.com/OpenLineage/OpenLineage/tree/main/integration/dbt"><code>dbt</code></a> -• <strong>Clients</strong> that send OpenLineage events to an HTTP backend. Both <a href="https://github.com/OpenLineage/OpenLineage/tree/main/client/java"><code>java</code></a> and <a href="https://github.com/OpenLineage/OpenLineage/tree/main/client/python"><code>python</code></a> are initially supported.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-13 18:45:37
 -
-

[OpenLineage/OpenLineage] Pull request opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/201|#201 Update integration.md>
-
- -
- -
-
Reviewers
- wslulciuc, collado-mike -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-13 18:51:26
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/202|#202 Append on RC or dev ver>
-
- <p>Always append <code>-SNAPSHOT</code> to the next development version.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-13 18:58:07
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/202|#202 Append on RC or dev ver>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-16 11:17:08
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/203|#203 [INTEGRATION] [dbt] strip model from job name, provide job namespace by env variable>
-
- <p>This PR strips <code>model.</code> from job names, since it's unnecessary - they are all prefixed by this.<br /> -Also, this PR provides possibility to override default job namespace by providing <code>OPENLINEAGE_NAMESPACE</code> env variable.</p> - -<p><a href="https://github.com/OpenLineage/OpenLineage/issues/193">#193</a><br /> -Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- -
-
Reviewers
- julienledem, wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-16 11:30:59
 -
-

[OpenLineage/OpenLineage] Pull request opened by OleksandrDvornik

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/204|#204 Introduce UnknownFacet for spark>
-
- <p>PR for <a href="https://github.com/OpenLineage/OpenLineage/issues/175">Issue:</a><br /> -Adding new custom facet for spark with name"spark_unknown".<br /> -Holds information about unvisited nodes from spark plan.</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-17 08:22:09
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/146|#146 Add dataQualityMetrics for airflow-GreatExpectations integration>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-17 08:22:23
 -
-

[OpenLineage/OpenLineage] Issue closed by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/145|#145 Add outputStatistics for dbt integration>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-17 11:29:35
 -
-

[OpenLineage/OpenLineage] Pull request merged by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/203|#203 [INTEGRATION] [dbt] strip model from job name, provide job namespace by env variable>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-17 12:05:29
 -
-

[OpenLineage/OpenLineage] Issue opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/205|#205 Preliminary work for Airflow 2.0>
-
- <p>☑︎ Test suite for 2.0 -☐ Working LineageBackend PoC that passes existing tests -☐ Issue in Airflow repo to create mechanism for reporting failures -☐ Way to add custom Extractors</p> -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-18 13:24:48
 -
-

[OpenLineage/OpenLineage] Pull request opened by mobuchowski

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/206|#206 [INTEGRATION] [dbt] parse dbt command line arguments that override defaults>
-
- <p>dbt run has few command line arguments that change location of artifacts, like <code>--project-dir</code> or ones that override defaults, like <code>--profile</code>. This PR adds support for few of them that can affect lineage event generation.</p> - -<p>Solves issue 2) in <a href="https://github.com/OpenLineage/OpenLineage/issues/193">#193</a><br /> -Signed-off-by: Maciej Obuchowski <a href="mailto:maciej.obuchowski@getindata.com">maciej.obuchowski@getindata.com</a></p> -
- -
-
Reviewers
- julienledem, wslulciuc -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-18 14:16:51
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/207|#207 [PROPOSAL] Model a BI dashboard in the OpenLineage model>
-
- <p><strong>Purpose:</strong><br /> -We want to track BI dashboards (Tableau, Looker, Superset, ...) in the lineage graph.<br /> -Like a <code>Job</code> a BI dashboard has a unique identifier and can change over time. It queries underlying datasets at irregular intervals given user interactions. Those are inputs to the BI dashboard.</p> - -<p><strong>Proposed implementation</strong><br /> -There are several ways to model this. We need to pick one.<br /> -A first proposal is as follows:</p> - -<p>• A dashboard is a <code>Job</code>, every time it issues queries, this creates a <code>Run</code> that captures the inputs (tables being queried).</p> -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-18 14:22:13
 -
-

[OpenLineage/OpenLineage] Issue opened by julienledem

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/issues/208|#208 [PROPOSAL] Add support for STAGE snowflake objects in the naming convention.>
-
- <p><strong>Purpose:</strong><br /> -In Snowflake, on can create a STAGE table before actually<br /> -<a href="https://docs.snowflake.com/en/sql-reference/sql/create-stage.html">https://docs.snowflake.com/en/sql-reference/sql/create-stage.html</a></p> - -<p><strong>Proposed implementation</strong><br /> -We need to update the naming spec to include the stage type in the table name to avoid conflict between a stage object and the final table.<br /> -<a href="https://github.com/OpenLineage/OpenLineage/blob/main/spec/Naming.md#snowflake">https://github.com/OpenLineage/OpenLineage/blob/main/spec/Naming.md#snowflake</a></p> - -<p>Current: <code>snowflake://{account name}/{database}.{schema}.{table}</code><br /> -Proposal: <code>snowflake://{account name}/{database}.{schema}.[STAGE.]{table}</code></p> -
- -
-
Labels
- proposal -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-18 17:38:09
 -
-

[OpenLineage/OpenLineage] Pull request opened by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/209|#209 Add maven badge>
-
- <p>This PR adds the maven badge displaying the latest release for <code>openlineage-java</code> and <code>openlineage-spark</code></p> -
- -
-
Reviewers
- julienledem -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
-
- - - - -
-
- ✅ All checks have passed -
- - - - - - - -
- - 1/1 successful checks -
- - -
- - - -
-
-
-
- - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-08-18 20:04:37
 -
-

[OpenLineage/OpenLineage] Pull request merged by wslulciuc

-
- - - - -
<https://github.com/OpenLineage/OpenLineage/pull/209|#209 Add maven badge>
-
- -
- - - - - - - -
- - <a href="https://github.com/OpenLineage/OpenLineage">OpenLineage/OpenLineage</a> -
- - -
- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Ross Turk - (ross@datakin.com) -
-
2021-09-03 23:41:38
 -
-

*Thread Reply:* If anyone has any more feedback on the website, feel free to add it to https://github.com/OpenLineage/website/issues 🙂

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
Willy Lulciuc - (willy@datakin.com) -
-
2021-10-01 15:17:07
 -
-

*Thread Reply:* @Julien Le Dem this ones ready for 👀

- - - -
-
-
-
- - - - - -
-
- - - - -
- -
Willy Lulciuc - (willy@datakin.com) -
-
2021-10-01 15:26:33
 -
-

*Thread Reply:* @Michael Collado I’ve also added you as a reviewer on the PR to get your thoughts

- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2021-11-15 12:30:14
 -
-

[OpenLineage/metrics] is now public!

-
- - -
- - - } - - OpenLineage - (https://github.com/OpenLineage) -
- - - -
-
- -
- - - - - - - - - -
- - <a href="https://github.com/OpenLineage/metrics">OpenLineage/metrics</a> -
- - -
- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2022-07-14 12:26:23
 -
-

[OpenLineage/docs] is now public!

-
- - -
- - - } - - OpenLineage - (https://github.com/OpenLineage) -
- - - -
-
- -
- - - - - - - - - -
- - <a href="https://github.com/OpenLineage/docs">OpenLineage/docs</a> -
- - -
- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
- -
GitHub - -
-
2023-11-20 16:40:42
 -
-

[OpenLineage/slack-archives] is now public!

-
- - -
- - - } - - OpenLineage - (https://github.com/OpenLineage) -
- - - -
-
- -
- - - - - - - - - -
- - <a href="https://github.com/OpenLineage/slack-archives">OpenLineage/slack-archives</a> -
- - -
- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - \ No newline at end of file