Merge pull request #45100 from Expensify/Rory-MapUpworkData

Map upwork data

package.json

@@ -267,6 +267,7 @@
     "copy-webpack-plugin": "^10.1.0",
     "css-loader": "^6.7.2",
     "csv-parse": "^5.5.5",
+    "csv-writer": "^1.6.0",
     "diff-so-fancy": "^1.3.0",
     "dotenv": "^16.0.3",
     "electron": "^29.4.6",

scripts/aggregateGitHubDataFromUpwork.ts

@@ -0,0 +1,175 @@
+/**
+ * This script is used for categorizing upwork costs into cost buckets for accounting purposes.
+ *
+ * To run this script from the root of E/App:
+ *
+ * ts-node ./scripts/aggregateGitHubDataFromUpwork.js <path_to_csv> <github_pat> <output_path>
+ *
+ * The input file must be a CSV with a single column containing just the GitHub issue number. The CSV must have a single header row.
+ */
+import {getOctokitOptions, GitHub} from '@actions/github/lib/utils';
+import {paginateRest} from '@octokit/plugin-paginate-rest';
+import {throttling} from '@octokit/plugin-throttling';
+import {createObjectCsvWriter} from 'csv-writer';
+import fs from 'fs';
+
+type OctokitOptions = {method: string; url: string; request: {retryCount: number}};
+type IssueType = 'bug' | 'feature' | 'other';
+
+if (process.argv.length < 3) {
+    throw new Error('Error: must provide filepath for CSV data');
+}
+
+if (process.argv.length < 4) {
+    throw new Error('Error: must provide GitHub token');
+}
+
+if (process.argv.length < 5) {
+    throw new Error('Error: must provide output file path');
+}
+
+// Get filepath for csv
+const inputFilepath = process.argv.at(2);
+if (!inputFilepath) {
+    throw new Error('Error: must provide filepath for CSV data');
+}
+
+// Get GitHub token
+const token = (process.argv.at(3) ?? '').trim();
+if (!token) {
+    throw new Error('Error: must provide GitHub token');
+}
+
+const Octokit = GitHub.plugin(throttling, paginateRest);
+const octokit = new Octokit(
+    getOctokitOptions(token, {
+        throttle: {
+            onRateLimit: (retryAfter: number, options: OctokitOptions) => {
+                console.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
+
+                // Retry once after hitting a rate limit error, then give up
+                if (options.request.retryCount <= 1) {
+                    console.log(`Retrying after ${retryAfter} seconds!`);
+                    return true;
+                }
+            },
+            onAbuseLimit: (retryAfter: number, options: OctokitOptions) => {
+                // does not retry, only logs a warning
+                console.warn(`Abuse detected for request ${options.method} ${options.url}`);
+            },
+        },
+    }),
+);
+
+// Get output filepath
+const outputFilepath = process.argv.at(4);
+if (!outputFilepath) {
+    throw new Error('Error: must provide output file path');
+}
+
+// Get data from csv
+const issues = fs
+    .readFileSync(inputFilepath)
+    .toString()
+    .split('\n')
+    .reduce((acc, issue) => {
+        if (!issue) {
+            return acc;
+        }
+        const issueNum = Number(issue.trim());
+        if (!issueNum) {
+            return acc;
+        }
+        acc.push(issueNum);
+        return acc;
+    }, [] as number[]);
+
+const csvWriter = createObjectCsvWriter({
+    path: outputFilepath,
+    header: [
+        {id: 'number', title: 'number'},
+        {id: 'title', title: 'title'},
+        {id: 'labels', title: 'labels'},
+        {id: 'type', title: 'type'},
+        {id: 'capSWProjects', title: 'capSWProjects'},
+    ],
+});
+
+function getIssueTypeFromLabels(labels: string[]): IssueType {
+    if (labels.includes('NewFeature')) {
+        return 'feature';
+    }
+    if (labels.includes('Bug')) {
+        return 'bug';
+    }
+    return 'other';
+}
+
+/**
+ * Returns a comma-delimited string with all projects associated with the given issue.
+ */
+async function getProjectsForIssue(issueNumber: number): Promise<string> {
+    const response = await octokit.graphql(
+        `
+        {
+                  repository(owner: "Expensify", name: "App") {
+                    issue(number: ${issueNumber}) {
+                      projectsV2(last: 30) {
+                        nodes {
+                          title
+                        }
+                      }
+                    }
+                  }
+                }
+        `,
+    );
+    return (response as {repository: {issue: {projectsV2: {nodes: Array<{title: string}>}}}}).repository.issue.projectsV2.nodes.map((node) => node.title).join(',');
+}
+
+async function getGitHubData() {
+    const gitHubData = [];
+    // Note: we fetch issues in a loop rather than in parallel to help address rate limiting issues with a PAT
+    for (const issueNumber of issues) {
+        console.info(`Fetching ${issueNumber}`);
+        const result = await octokit.rest.issues
+            .get({
+                owner: 'Expensify',
+                repo: 'App',
+                // eslint-disable-next-line @typescript-eslint/naming-convention
+                issue_number: issueNumber,
+            })
+            .catch(() => {
+                console.warn(`Error getting issue ${issueNumber}`);
+            });
+        if (result) {
+            const issue = result.data;
+            const labels = issue.labels.reduce((acc, label) => {
+                if (typeof label === 'string') {
+                    acc.push(label);
+                } else if (label.name) {
+                    acc.push(label.name);
+                }
+                return acc;
+            }, [] as string[]);
+            const type = getIssueTypeFromLabels(labels);
+            let capSWProjects = '';
+            if (type === 'feature') {
+                // eslint-disable-next-line @typescript-eslint/no-unsafe-call
+                capSWProjects = await getProjectsForIssue(issueNumber);
+            }
+            gitHubData.push({
+                number: issue.number,
+                title: issue.title,
+                labels,
+                type,
+                capSWProjects,
+            });
+        }
+    }
+    return gitHubData;
+}
+
+getGitHubData()
+    .then((gitHubData) => csvWriter.writeRecords(gitHubData))
+    .then(() => console.info(`Done ✅ Wrote file to ${outputFilepath}`));