Add Airflow SmartSuggest [Don't merge]

airflow/www/static/js/api/index.ts

@@ -51,6 +51,9 @@ import useDagRuns from "./useDagRuns";
 import useHistoricalMetricsData from "./useHistoricalMetricsData";
 import { useTaskXcomEntry, useTaskXcomCollection } from "./useTaskXcom";
 import useEventLogs from "./useEventLogs";
+import useTaskAnomaliesData from "./useTaskAnomaliesData";
+import useDagAnomaliesData from "./useDagAnomaliesData";
+import useResourceAnomaliesData from "./useResourceAnomaliesData";
 
 axios.interceptors.request.use((config) => {
   config.paramsSerializer = {
@@ -98,4 +101,7 @@ export {
   useTaskXcomEntry,
   useTaskXcomCollection,
   useEventLogs,
+  useTaskAnomaliesData,
+  useDagAnomaliesData,
+  useResourceAnomaliesData,
 };

airflow/www/static/js/api/useDagAnomaliesData.ts

@@ -0,0 +1,40 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import axios, { AxiosResponse } from "axios";
+import { useQuery } from "react-query";
+import type { DagAnomaliesData } from "src/types";
+
+import { getMetaValue } from "src/utils";
+
+const url = getMetaValue("dag_anomalies_data_url");
+
+const useDagAnomaliesData = (startDate: string, endDate: string) =>
+  useQuery(
+    ["dag_anomalies_data", startDate, endDate],
+    async () =>
+      axios.get<AxiosResponse, DagAnomaliesData>(url, {
+        params: { start_date: startDate, end_date: endDate },
+      }),
+    {
+      refetchInterval: 60 * 1000,
+    }
+  );
+
+export default useDagAnomaliesData;

airflow/www/static/js/api/useResourceAnomaliesData.ts

@@ -0,0 +1,40 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import axios, { AxiosResponse } from "axios";
+import { useQuery } from "react-query";
+import type { ResourceAnomaliesData } from "src/types";
+
+import { getMetaValue } from "src/utils";
+
+const url = getMetaValue("resource_anomalies_data_url");
+
+const useResourceAnomaliesData = (startDate: string, endDate: string) =>
+  useQuery(
+    ["resource_anomalies_data", startDate, endDate],
+    async () =>
+      axios.get<AxiosResponse, ResourceAnomaliesData>(url, {
+        params: { start_date: startDate, end_date: endDate },
+      }),
+    {
+      refetchInterval: 60 * 1000,
+    }
+  );
+
+export default useResourceAnomaliesData;

airflow/www/static/js/api/useTaskAnomaliesData.ts

@@ -0,0 +1,40 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import axios, { AxiosResponse } from "axios";
+import { useQuery } from "react-query";
+import type { TaskAnomaliesData } from "src/types";
+
+import { getMetaValue } from "src/utils";
+
+const url = getMetaValue("task_anomalies_data_url");
+
+const useTaskAnomaliesData = (startDate: string, endDate: string) =>
+  useQuery(
+    ["historical_metrics_data", startDate, endDate],
+    async () =>
+      axios.get<AxiosResponse, TaskAnomaliesData>(url, {
+        params: { start_date: startDate, end_date: endDate },
+      }),
+    {
+      refetchInterval: (autoRefreshInterval || 1) * 1000,
+    }
+  );
+
+export default useTaskAnomaliesData;

airflow/www/static/js/cluster-activity/dag-anomalies/PieChart.tsx

@@ -0,0 +1,147 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React from "react";
+import {
+  Box,
+  BoxProps,
+  Card,
+  CardBody,
+  CardHeader,
+  Heading,
+  useTheme,
+} from "@chakra-ui/react";
+import ReactECharts, { ReactEChartsProps } from "src/components/ReactECharts";
+import type { DagAnomaliesData } from "src/types";
+import { camelCase, mapKeys } from "lodash";
+
+interface SeriesPoint {
+  name: string;
+  value: number;
+}
+
+type SeriesData = Array<SeriesPoint>;
+
+const camelCaseColorPalette = mapKeys(stateColors, (_, k) => camelCase(k));
+
+const formatData = (
+  data: DagAnomaliesData[keyof DagAnomaliesData] | undefined
+): [number, SeriesData] => {
+  if (data === undefined) return [0, []];
+
+  let sum = 0;
+  const formattedData: { name: string; value: number }[] = [];
+  Object.entries(data).forEach(([k, v]) => {
+    sum += v;
+    formattedData.push({
+      name: k,
+      value: v,
+    });
+  });
+  formattedData.sort((a: SeriesPoint, b: SeriesPoint) => b.value - a.value);
+  return [sum, formattedData];
+};
+
+interface Props extends BoxProps {
+  title: string;
+  data?: DagAnomaliesData[keyof DagAnomaliesData];
+  colorPalette?: {
+    [key: string]: string;
+  };
+}
+
+const PieChart = ({
+  title,
+  data,
+  colorPalette = camelCaseColorPalette,
+  ...rest
+}: Props) => {
+  const theme = useTheme();
+  const [sum, formattedData] = formatData(data);
+  const option: ReactEChartsProps["option"] = {
+    title: {
+      text: `on a total of ${sum}`,
+      left: "right",
+      top: "bottom",
+      textStyle: {
+        fontSize: "14px",
+        color: theme.colors.gray["500"],
+      },
+    },
+    tooltip: {
+      trigger: "item",
+    },
+    legend: {
+      left: "center",
+      type: "scroll",
+    },
+    color: formattedData?.map((d) => {
+      let color = colorPalette[d.name];
+      if (color === undefined) {
+        // eslint-disable-next-line no-console
+        console.warn(
+          `The color for ${d.name} is missing from the palette, defaulting to black`
+        );
+        color = "black";
+      }
+      return color;
+    }),
+    series: [
+      {
+        name: title,
+        type: "pie",
+        radius: ["35%", "60%"],
+        avoidLabelOverlap: false,
+        top: "0%",
+        itemStyle: {
+          borderRadius: 5,
+          borderColor: "#fff",
+          borderWidth: 2,
+        },
+        label: {
+          show: false,
+          position: "center",
+        },
+        emphasis: {
+          label: {
+            show: true,
+            fontSize: 16,
+            fontWeight: "bold",
+          },
+        },
+        data: formattedData,
+      },
+    ],
+  };
+
+  return (
+    <Box {...rest}>
+      <Card h="100%">
+        <CardHeader textAlign="center" p={3}>
+          <Heading size="md">{title}</Heading>
+        </CardHeader>
+        <CardBody>
+          <ReactECharts option={option} />
+        </CardBody>
+      </Card>
+    </Box>
+  );
+};
+
+export default PieChart;

airflow/www/static/js/cluster-activity/dag-anomalies/index.tsx

@@ -0,0 +1,97 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React from "react";
+import {
+  Card,
+  CardBody,
+  CardHeader,
+  Heading,
+  Text,
+  Flex,
+  Table,
+  Tbody,
+  Tr,
+  Td,
+  Code,
+  Box,
+  Thead,
+  Th,
+} from "@chakra-ui/react";
+import InfoTooltip from "src/components/InfoTooltip";
+import FilterBar from "src/cluster-activity/nav/FilterBar";
+import useFilters from "src/cluster-activity/useFilters";
+import { useDagAnomaliesData } from "src/api";
+import PieChart from "src/cluster-activity/historical-metrics/PieChart";
+import LoadingWrapper from "src/components/LoadingWrapper";
+import { SimpleStatus } from "src/dag/StatusBox";
+import { ClipboardText } from "src/components/Clipboard";
+import { formatDuration, getDuration } from "src/datetime_utils";
+import Time from "src/components/Time";
+
+const DagAnomalies = () => {
+  const {
+    filters: { startDate, endDate },
+  } = useFilters();
+  const { data, isError } = useDagAnomaliesData(startDate, endDate);
+  return (
+    <Flex w="100%">
+      <Card w="100%">
+        <CardHeader>
+          <Flex alignItems="center">
+            <Heading size="md">Dag Anomalies</Heading>
+            {/* <InfoTooltip */}
+            {/*  label="Based on historical data. You can adjust the period by setting a different start and end date filter." */}
+            {/*  size={18} */}
+            {/* /> */}
+          </Flex>
+        </CardHeader>
+        <CardBody>
+          {/* <FilterBar /> */}
+          <Flex justifyContent="center" minH="200px" alignItems="center">
+            <LoadingWrapper hasData={!!data} isError={isError}>
+              <Flex flexWrap="wrap" width="100%">
+                <Table variant="striped">
+                  <Thead>
+                    <Tr>
+                      <Th>Dag Id</Th>
+                      <Th>Duration</Th>
+                    </Tr>
+                  </Thead>
+                  <Tbody>
+                    {(data?.dagAnomalies || []).map((d) => (
+                      <Tr key={d.dagId}>
+                        <Td>{d.dagId}</Td>
+                        <Td>
+                          <Code fontSize="md">{d.reason}</Code>
+                        </Td>
+                      </Tr>
+                    ))}
+                  </Tbody>
+                </Table>
+              </Flex>
+            </LoadingWrapper>
+          </Flex>
+        </CardBody>
+      </Card>
+    </Flex>
+  );
+};
+
+export default DagAnomalies;