Merge pull request #330 from populationgenomics/upstream-13892

Merge upstream HEAD(b7bde56, 2024-05-14) Stop writing to V2 tables

.git-blame-ignore-revs

@@ -8,3 +8,9 @@
 4fccbe2d18c6d2f4059036d61489467c780bbc0e
 # Delete `FastIndexedSeq`
 fcc9ffab3cd68c4cfb26a1553d65118797c59d6e
+# blackens hail directory
+da2790242a40ec425a53a02707d261c893b264f7
+# scalafmt
+422edf6386616711ca70f87c455f76781ac925d4
+# replaces black formatting with ruff
+fa2ef0f2c76654d0c037ff6db60ccb8842fb8539

.pre-commit-config.yaml

@@ -9,24 +9,26 @@ repos:
         language: system
         types: [python]
         require_serial: true
+      - id: ruff-format
+        name: ruff-format
+        entry: ruff format
+        language: system
+        types: [python]
+        require_serial: true
       - id: pyright
         name: pyright
         entry: pyright
         language: system
         types: [python]
         require_serial: true
+        exclude: hail/python/(hail|test)
         stages:
           - pre-push
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.3.0
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
-    hooks:
-      - id: black
-        language_version: python3
   - repo: https://github.com/thibaudcolas/curlylint
     rev: v0.13.1
     hooks:

Makefile

@@ -41,6 +41,7 @@ check-all: check-hail check-services
 check-hail-fast:
 	ruff check hail/python/hail
 	ruff check hail/python/hailtop
+	ruff format hail --diff
 	$(PYTHON) -m pyright hail/python/hailtop
 
 .PHONY: pylint-hailtop
@@ -50,6 +51,7 @@ pylint-hailtop:
 
 .PHONY: check-hail
 check-hail: check-hail-fast pylint-hailtop
+	cd hail && sh ./gradlew spotlessCheck
 
 .PHONY: check-services
 check-services: $(CHECK_SERVICES_MODULES)
@@ -61,8 +63,8 @@ pylint-%:
 .PHONY: check-%-fast
 check-%-fast:
 	ruff check $*
+	ruff format $* --diff
 	$(PYTHON) -m pyright $*
-	$(PYTHON) -m black $* --check --diff
 	curlylint $*
 	cd $* && bash ../check-sql.sh
 
@@ -173,7 +175,8 @@ hail-0.1-docs-5a6778710097.tar.gz:
 	gcloud storage cp gs://hail-common/builds/0.1/docs/$@ .
 
 hail/build/www: hail-0.1-docs-5a6778710097.tar.gz $(shell git ls-files hail)
-	$(MAKE) -C hail hail-docs-no-test batch-docs
+	@echo !!! This target does not render the notebooks because it takes a long time !!!
+	$(MAKE) -C hail hail-docs-do-not-render-notebooks batch-docs
 	mkdir -p hail/build/www/docs/0.1
 	tar -xvf hail-0.1-docs-5a6778710097.tar.gz -C hail/build/www/docs/0.1 --strip-components 2
 	touch $@  # Copying into the dir does not necessarily touch it

auth/auth/auth.py

@@ -164,10 +164,10 @@ async def _insert(tx):
             return False
 
         await tx.execute_insertone(
-            '''
+            """
 INSERT INTO users (state, username, login_id, is_developer, is_service_account, hail_identity, hail_credentials_secret_name)
 VALUES (%s, %s, %s, %s, %s, %s, %s);
-''',
+""",
             (
                 'creating',
                 username,
@@ -482,9 +482,11 @@ async def rest_login(request: web.Request) -> web.Response:
     flow_data['callback_uri'] = callback_uri
 
     # keeping authorization_url and state for backwards compatibility
-    return json_response(
-        {'flow': flow_data, 'authorization_url': flow_data['authorization_url'], 'state': flow_data['state']}
-    )
+    return json_response({
+        'flow': flow_data,
+        'authorization_url': flow_data['authorization_url'],
+        'state': flow_data['state'],
+    })
 
 
 @routes.get('/api/v1alpha/oauth2-client')
@@ -511,10 +513,10 @@ async def post_create_role(request: web.Request, _) -> NoReturn:
     name = str(post['name'])
 
     role_id = await db.execute_insertone(
-        '''
+        """
 INSERT INTO `roles` (`name`)
 VALUES (%s);
-''',
+""",
         (name),
     )
 
@@ -564,10 +566,10 @@ async def rest_get_users(request: web.Request, userdata: UserData) -> web.Respon
         raise web.HTTPUnauthorized()
 
     db = request.app[AppKeys.DB]
-    _query = '''
+    _query = """
 SELECT id, username, login_id, state, is_developer, is_service_account, hail_identity
 FROM users;
-'''
+"""
     users = [x async for x in db.select_and_fetchall(_query)]
     return json_response(users)
 
@@ -579,10 +581,10 @@ async def rest_get_user(request: web.Request, _) -> web.Response:
     username = request.match_info['user']
 
     user = await db.select_and_fetchone(
-        '''
+        """
 SELECT id, username, login_id, state, is_developer, is_service_account, hail_identity FROM users
 WHERE username = %s;
-''',
+""",
         (username,),
     )
     if user is None:
@@ -599,11 +601,11 @@ async def _delete_user(db: Database, username: str, id: Optional[str]):
         where_args.append(id)
 
     n_rows = await db.execute_update(
-        f'''
+        f"""
 UPDATE users
 SET state = 'deleting'
 WHERE {' AND '.join(where_conditions)};
-''',
+""",
         where_args,
     )
 
@@ -743,11 +745,11 @@ async def get_userinfo_from_login_id_or_hail_identity_id(
     users = [
         x
         async for x in db.select_and_fetchall(
-            '''
+            """
 SELECT users.*
 FROM users
 WHERE (users.login_id = %s OR users.hail_identity_uid = %s) AND users.state = 'active'
-''',
+""",
             (login_id_or_hail_idenity_uid, login_id_or_hail_idenity_uid),
         )
     ]
@@ -767,12 +769,12 @@ async def get_userinfo_from_hail_session_id(request: web.Request, session_id: st
     users = [
         x
         async for x in db.select_and_fetchall(
-            '''
+            """
 SELECT users.*
 FROM users
 INNER JOIN sessions ON users.id = sessions.user_id
 WHERE users.state = 'active' AND sessions.session_id = %s AND (ISNULL(sessions.max_age_secs) OR (NOW() < TIMESTAMPADD(SECOND, sessions.max_age_secs, sessions.created)));
-''',
+""",
             session_id,
             'get_userinfo',
         )

auth/auth/driver/driver.py

@@ -94,10 +94,10 @@ async def delete(self):
             return
 
         await self.db.just_execute(
-            '''
+            """
 DELETE FROM sessions
 WHERE session_id = %s;
-''',
+""",
             (self.session_id,),
         )
         self.session_id = None
@@ -430,11 +430,11 @@ async def _create_user(app, user, skip_trial_bp, cleanup):
             updates['trial_bp_name'] = billing_project_name
 
     n_rows = await db.execute_update(
-        f'''
+        f"""
 UPDATE users
 SET {', '.join([f'{k} = %({k})s' for k in updates])}
 WHERE id = %(id)s AND state = 'creating';
-''',
+""",
         {'id': user['id'], **updates},
     )
     if n_rows != 1:
@@ -504,10 +504,10 @@ async def delete_user(app, user):
         await bp.delete()
 
     await db.just_execute(
-        '''
+        """
 DELETE FROM sessions WHERE user_id = %s;
 UPDATE users SET state = 'deleted' WHERE id = %s;
-''',
+""",
         (user['id'], user['id']),
     )
 
@@ -525,11 +525,11 @@ async def resolve_identity_uid(app, hail_identity):
         hail_identity_uid = await sp.get_service_principal_object_id()
 
     await db.just_execute(
-        '''
+        """
 UPDATE users
 SET hail_identity_uid = %s
 WHERE hail_identity = %s
-''',
+""",
         (hail_identity_uid, hail_identity),
     )
 

auth/auth/exceptions.py

@@ -7,7 +7,7 @@ def __init__(self, message, severity):
         self.message = message
         self.ui_error_type = severity
 
-    def http_response(self):
+    def http_response(self) -> web.HTTPError:
         return web.HTTPBadRequest(reason=self.message)
 
 

auth/deployment.yaml

@@ -45,11 +45,6 @@ spec:
             value: "{{ default_ns.name }}"
           - name: HAIL_DEPLOY_CONFIG_FILE
             value: /deploy-config/deploy-config.json
-          - name: HAIL_DOMAIN
-            valueFrom:
-              secretKeyRef:
-                name: global-config
-                key: domain
           - name: CLOUD
             valueFrom:
               secretKeyRef:
@@ -179,11 +174,6 @@ spec:
             value: "{{ default_ns.name }}"
           - name: HAIL_DEPLOY_CONFIG_FILE
             value: /deploy-config/deploy-config.json
-          - name: HAIL_DOMAIN
-            valueFrom:
-              secretKeyRef:
-                name: global-config
-                key: domain
           - name: HAIL_ORGANIZATION_DOMAIN
             valueFrom:
               secretKeyRef:

batch/Dockerfile.worker

@@ -51,7 +51,7 @@ RUN hail-pip-install \
     -r hailtop-requirements.txt \
     -r gear-requirements.txt \
     -r batch-requirements.txt \
-    pyspark==3.3.0
+    pyspark==3.3.2
 
 ENV SPARK_HOME /usr/local/lib/python3.9/dist-packages/pyspark
 ENV PATH "$PATH:$SPARK_HOME/sbin:$SPARK_HOME/bin"

batch/batch/batch.py

@@ -114,11 +114,11 @@ async def cancel_batch_in_db(db, batch_id):
     @transaction(db)
     async def cancel(tx):
         record = await tx.execute_and_fetchone(
-            '''
+            """
 SELECT `state` FROM batches
 WHERE id = %s AND NOT deleted
 FOR UPDATE;
-''',
+""",
             (batch_id,),
         )
         if not record:

batch/batch/cloud/azure/driver/create_instance.py

@@ -92,7 +92,7 @@ def create_vm_config(
 
     jvm_touch_command = '\n'.join(touch_commands)
 
-    startup_script = r'''#cloud-config
+    startup_script = r"""#cloud-config
 
 mounts:
   - [ ephemeral0, null ]
@@ -123,10 +123,10 @@ def create_vm_config(
 
 runcmd:
   - sh /startup.sh
-'''
+"""
     startup_script = base64.b64encode(startup_script.encode('utf-8')).decode('utf-8')
 
-    run_script = f'''
+    run_script = f"""
 #!/bin/bash
 set -x
 
@@ -302,7 +302,7 @@ def create_vm_config(
 az vm delete -g $RESOURCE_GROUP -n $NAME --yes
 sleep 1
 done
-'''
+"""
 
     user_data = {
         'run_script': run_script,

batch/batch/cloud/azure/driver/driver.py

@@ -37,10 +37,10 @@ async def create(
 
         region_args = [(r,) for r in regions]
         await db.execute_many(
-            '''
+            """
 INSERT INTO regions (region) VALUES (%s)
 ON DUPLICATE KEY UPDATE region = region;
-''',
+""",
             region_args,
         )
 

batch/batch/cloud/azure/instance_config.py

@@ -35,16 +35,14 @@ def create(
         else:
             data_disk_resource = AzureStaticSizedDiskResource.create(product_versions, 'P', data_disk_size_gb, location)
 
-        resources: List[AzureResource] = filter_none(
-            [
-                AzureVMResource.create(product_versions, machine_type, preemptible, location),
-                AzureStaticSizedDiskResource.create(product_versions, 'E', boot_disk_size_gb, location),
-                data_disk_resource,
-                AzureDynamicSizedDiskResource.create(product_versions, 'P', location),
-                AzureIPFeeResource.create(product_versions, 1024),
-                AzureServiceFeeResource.create(product_versions),
-            ]
-        )
+        resources: List[AzureResource] = filter_none([
+            AzureVMResource.create(product_versions, machine_type, preemptible, location),
+            AzureStaticSizedDiskResource.create(product_versions, 'E', boot_disk_size_gb, location),
+            data_disk_resource,
+            AzureDynamicSizedDiskResource.create(product_versions, 'P', location),
+            AzureIPFeeResource.create(product_versions, 1024),
+            AzureServiceFeeResource.create(product_versions),
+        ])
 
         return AzureSlimInstanceConfig(
             machine_type=machine_type,