From 2b143c258abb22a4a769fe66f11458e5590236a1 Mon Sep 17 00:00:00 2001 From: Qubad786 Date: Wed, 9 May 2018 19:58:15 +0500 Subject: [PATCH] revert 3rd party transcripts old workflows --- control/old_veda_deliver_cielo.py | 171 ------------------------------ control/veda_deliver.py | 69 ------------ instance_config.yaml | 5 - static_config.yaml | 3 - 4 files changed, 248 deletions(-) delete mode 100644 control/old_veda_deliver_cielo.py diff --git a/control/old_veda_deliver_cielo.py b/control/old_veda_deliver_cielo.py deleted file mode 100644 index 5f6fa44..0000000 --- a/control/old_veda_deliver_cielo.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -Cielo24 API Job Start and Download -Options (reflected in Course.models): -transcription_fidelity = - Mechanical (75%), - Premium (95%)(3-72h), - Professional (99+%)(3-72h) -priority = - standard (24h), - priority (48h) -turnaround_hours = number, overrides 'priority' call, will change a standard to a priority silently -""" -import logging -import requests -import ast -import urllib - -from control_env import * - -requests.packages.urllib3.disable_warnings() -LOGGER = logging.getLogger(__name__) -# TODO: Remove this temporary logging to stdout -logging.basicConfig(stream=sys.stdout, level=logging.INFO) - - -class Cielo24TranscriptOld(object): - - def __init__(self, veda_id): - self.veda_id = veda_id - '''Defaults''' - self.c24_site = 'https://api.cielo24.com/api' - self.c24_login = '/account/login' - self.c24_joblist = '/job/list' - self.c24_newjob = '/job/new' - self.add_media = '/job/add_media' - self.transcribe = '/job/perform_transcription' - - '''Retreive C24 Course-based defaults''' - self.c24_defaults = self.retrieve_defaults() - - def perform_transcription(self): - if self.c24_defaults['c24_user'] is None: - return None - ''' - GET /api/job/perform_transcription?v=1 HTTP/1.1 - &api_token=xxxx - &job_id=xxxx - &transcription_fidelity=PREMIUM&priority=STANDARD - Host: api.cielo24.com - ''' - api_token = self.tokengenerator() - if api_token is None: - return None - - job_id = self.generate_jobs(api_token) - task_id = self.embed_url(api_token, job_id) - - r5 = requests.get( - ''.join(( - self.c24_site, - self.transcribe, - '?v=1&api_token=', - api_token, - '&job_id=', - job_id, - '&transcription_fidelity=', - self.c24_defaults['c24_fidelity'], - '&priority=', - self.c24_defaults['c24_speed'] - )) - ) - return ast.literal_eval(r5.text)['TaskId'] - - def retrieve_defaults(self): - video_query = Video.objects.filter( - edx_id=self.veda_id - ).latest() - - url_query = URL.objects.filter( - videoID=video_query, - encode_url__icontains='_DTH.mp4', - ).latest() - - if video_query.inst_class.c24_username is None: - LOGGER.error('[VIDEO_PIPELINE] {id} : Cielo API : Course record incomplete'.format(id=self.veda_id)) - return None - - c24_defaults = { - 'c24_user': video_query.inst_class.c24_username, - 'c24_pass': video_query.inst_class.c24_password, - 'c24_speed': video_query.inst_class.c24_speed, - 'c24_fidelity': video_query.inst_class.c24_fidelity, - 'edx_id': self.veda_id, - 'url': url_query.encode_url - } - return c24_defaults - - def tokengenerator(self): - token_url = self.c24_site + self.c24_login + \ - '?v=1&username=' + self.c24_defaults['c24_user'] + \ - '&password=' + self.c24_defaults['c24_pass'] - - # Generate Token - r1 = requests.get(token_url) - if r1.status_code > 299: - LOGGER.error('[VIDEO_PIPELINE] {id} : Cielo API access'.format(id=self.veda_id)) - return - api_token = ast.literal_eval(r1.text)["ApiToken"] - return api_token - - def listjobs(self): - """List Jobs""" - api_token = self.tokengenerator() - r2 = requests.get( - ''.join(( - self.c24_site, - self.c24_joblist, - '?v=1&api_token=', - api_token - )) - ) - job_list = r2.text - return job_list - - def generate_jobs(self, api_token): - """ - 'https://api.cielo24.com/job/new?v=1&\ - api_token=xxx&job_name=xxx&language=en' - """ - r3 = requests.get( - ''.join(( - self.c24_site, - self.c24_newjob, - '?v=1&api_token=', - api_token, - '&job_name=', - self.c24_defaults['edx_id'], - '&language=en' - )) - ) - job_id = ast.literal_eval(r3.text)['JobId'] - return job_id - - def embed_url(self, api_token, job_id): - """ - GET /api/job/add_media?v=1&api_token=xxxx - &job_id=xxxxx - &media_url=http%3A%2F%2Fwww.domain.com%2Fvideo.mp4 HTTP/1.1 - Host: api.cielo24.com - """ - r4 = requests.get( - ''.join(( - self.c24_site, - self.add_media, - '?v=1&api_token=', - api_token, - '&job_id=', - job_id, - '&media_url=', - urllib.quote_plus(self.c24_defaults['url']) - )) - ) - return ast.literal_eval(r4.text)['TaskId'] - - -def main(): - pass - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/control/veda_deliver.py b/control/veda_deliver.py index 9df97d1..2d96ab9 100644 --- a/control/veda_deliver.py +++ b/control/veda_deliver.py @@ -5,7 +5,6 @@ """ import datetime -import ftplib import logging import shutil from os.path import expanduser @@ -19,7 +18,6 @@ from boto.s3.key import Key from django.core.urlresolvers import reverse -from control.old_veda_deliver_cielo import Cielo24TranscriptOld from control_env import * from veda_deliver_cielo import Cielo24Transcript from veda_deliver_youtube import DeliverYoutube @@ -129,11 +127,6 @@ def run(self): u1.encode_size = self.video_proto.filesize u1.save() - # TODO: Warning! this shall be removed once 3rd party credentials - # for existing courses are migrated according to new flow. - self._THREEPLAY_UPLOAD() - self._CIELO24_UPLOAD() - self.status = self._DETERMINE_STATUS() self._UPDATE_DATA() self._CLEANUP() @@ -624,68 +617,6 @@ def start_3play_transcription_process(self, encoded_file): self.video_query.studio_id, ) - def _CIELO24_UPLOAD(self): - """ - Note: This is part of the old flow which was deprecated when transcript phase 1 was reased. - """ - # TODO: this must be removed once existing 3rd party credentials are migrated according - # to the new workflow. - if self.video_query.inst_class.c24_proc is False: - return None - - if self.encode_profile != 'desktop_mp4': - return None - - cielojob = Cielo24TranscriptOld( - veda_id=self.video_query.edx_id - ) - cielojob.perform_transcription() - LOGGER.info('[DELIVERY] {id} : Cielo24 job sent '.format(id=self.video_query.edx_id)) - - def _THREEPLAY_UPLOAD(self): - """ - Note: This is part of the old flow which was deprecated when transcript phase 1 was reased. - """ - # TODO: this must be removed once existing 3rd party credentials are migrated according - # to the new workflow. - if self.video_query.inst_class.tp_proc is False: - return None - - if self.encode_profile != 'desktop_mp4': - return None - - ftp1 = ftplib.FTP( - self.auth_dict['threeplay_ftphost'] - ) - user = self.video_query.inst_class.tp_username.strip() - passwd = self.video_query.inst_class.tp_password.strip() - try: - ftp1.login(user, passwd) - except: - LOGGER.error('[DELIVERY] {file} : 3Play Authentication Failure'.format(file=self.encoded_file)) - try: - ftp1.cwd( - self.video_query.inst_class.tp_speed - ) - except: - ftp1.mkd( - self.video_query.inst_class.tp_speed - ) - ftp1.cwd( - self.video_query.inst_class.tp_speed - ) - os.chdir(self.node_work_directory) - - ftp1.storbinary( - 'STOR ' + self.encoded_file, - open(os.path.join( - self.node_work_directory, - self.encoded_file - ), 'rb') - ) - - os.chdir(homedir) - def YOUTUBE_SFTP(self, review=False): if self.video_query.inst_class.yt_proc is False: if self.video_query.inst_class.review_proc is False: diff --git a/instance_config.yaml b/instance_config.yaml index d426a86..a2f2229 100644 --- a/instance_config.yaml +++ b/instance_config.yaml @@ -114,9 +114,4 @@ sg_server_path: sg_script_name: sg_script_key: -# --- -# Endpoints -# --- -threeplay_ftphost: - ... diff --git a/static_config.yaml b/static_config.yaml index 379f7db..12b277d 100644 --- a/static_config.yaml +++ b/static_config.yaml @@ -1,9 +1,6 @@ --- # This configuration should only have static settings. -# 3PlayMedia FTP host for old workflow -threeplay_ftphost: ftp.3playmedia.com - # s3 bucket static prefixes edx_s3_processed_prefix: prod-edx/processed/ edx_s3_rejected_prefix: prod-edx/rejected/