diff --git a/README.md b/README.md index 083ef02..2ed7098 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ I've downloaded many other good videos such as those from Khan Academy. certain resources. * File format extension filter to grab resource types you want. * Login credentials accepted on command-line or from `.netrc` file. + * Default arguments loaded from `coursera-dl.conf` file. * Core functionality tested on Linux, Mac and Windows. # Disclaimer @@ -278,6 +279,23 @@ instead. This is especially convenient, as typing usernames (email addresses) and passwords directly on the command line can get tiresome (even more if you happened to choose a "strong" password). +Alternatively, if you want to store your preferred parameters (which might +also include your username and password), create a file named `coursera-dl.conf` +where the script is supposed to be executed, with the following format: + + --username + --password + --subtitle-language en,zh-CN|zh-TW + --download-quizzes True + #--mathjax-cdn https://cdn.bootcss.com/mathjax/2.7.1/MathJax.js + # more other parameters + +Parameter which is stored in the file will be overriden if it is again specifed +in your commandline script + +**Note:** In `coursera-dl.conf`, all the parameters should not be wrapped +with quotes. + ## Resuming downloads In default mode when you interrupt the download process by pressing @@ -341,7 +359,7 @@ one of the following actions solve your problem: * If results show 0 sections, you most likely have provided invalid credentials (username and/or password in the command line or in your - `.netrc` file). + `.netrc` file or in your `coursera-dl.conf` file). * For courses that have not started yet, but have had a previous iteration sometimes a preview is available, containing all the classes from the last @@ -456,6 +474,15 @@ If you still have the problem, please read the following issues for more ideas o This is also worth reading: https://urllib3.readthedocs.io/en/latest/security.html#insecureplatformwarning +## Use an alternative cdn url for `MathJax.js` + +When saving a course page, we enabled `MathJax` rendering for math equations, by +injecting `MathJax.js` in the header. The script is using a cdn service provided +by [mathjax.org](https://cdn.mathjax.org/mathjax/latest/MathJax.js). However, that +url is not accessible in some countries/regions, you can provide a +`--mathjax-cdn ` parameter to specify the `MathJax.js` file that is +accessible in your region. + # Reporting issues Before reporting any issue please follow the steps below: diff --git a/coursera/api.py b/coursera/api.py index e22c32d..a680b1a 100644 --- a/coursera/api.py +++ b/coursera/api.py @@ -25,12 +25,18 @@ from .define import (OPENCOURSE_SUPPLEMENT_URL, OPENCOURSE_ONDEMAND_COURSE_MATERIALS, OPENCOURSE_VIDEO_URL, OPENCOURSE_MEMBERSHIPS, + OPENCOURSE_REFERENCES_POLL_URL, + OPENCOURSE_REFERENCE_ITEM_URL, + OPENCOURSE_PROGRAMMING_IMMEDIATE_INSTRUCTIOINS_URL, + POST_OPENCOURSE_API_QUIZ_SESSION, POST_OPENCOURSE_API_QUIZ_SESSION_GET_STATE, POST_OPENCOURSE_ONDEMAND_EXAM_SESSIONS, POST_OPENCOURSE_ONDEMAND_EXAM_SESSIONS_GET_STATE, - INSTRUCTIONS_HTML_INJECTION, + INSTRUCTIONS_HTML_INJECTION_PRE, + INSTRUCTIONS_HTML_MATHJAX_URL, + INSTRUCTIONS_HTML_INJECTION_AFTER, IN_MEMORY_EXTENSION, IN_MEMORY_MARKER) @@ -135,9 +141,12 @@ class QuizExamToMarkupConverter(object): class MarkupToHTMLConverter(object): - def __init__(self, session): + def __init__(self, session, mathjax_cdn_url=None): self._session = session self._asset_retriever = AssetRetriever(session) + if not mathjax_cdn_url: + mathjax_cdn_url = INSTRUCTIONS_HTML_MATHJAX_URL + self._mathjax_cdn_url = mathjax_cdn_url def __call__(self, markup): """ @@ -170,7 +179,11 @@ class MarkupToHTMLConverter(object): soup.insert(0, meta) # 1. Inject basic CSS style - css_soup = BeautifulSoup(INSTRUCTIONS_HTML_INJECTION) + css = "".join([ + INSTRUCTIONS_HTML_INJECTION_PRE, + self._mathjax_cdn_url, + INSTRUCTIONS_HTML_INJECTION_AFTER]) + css_soup = BeautifulSoup(css) soup.append(css_soup) # 2. Replace with

@@ -386,7 +399,8 @@ class CourseraOnDemand(object): """ def __init__(self, session, course_id, course_name, - unrestricted_filenames=False): + unrestricted_filenames=False, + mathjax_cdn_url=None): """ Initialize Coursera OnDemand API. @@ -409,7 +423,7 @@ class CourseraOnDemand(object): self._user_id = None self._quiz_to_markup = QuizExamToMarkupConverter(session) - self._markup_to_html = MarkupToHTMLConverter(session) + self._markup_to_html = MarkupToHTMLConverter(session, mathjax_cdn_url=mathjax_cdn_url) self._asset_retriever = AssetRetriever(session) def obtain_user_id(self): @@ -721,29 +735,11 @@ class CourseraOnDemand(object): video_url = sources[0]['formatSources']['video/mp4'] video_content['mp4'] = video_url - # subtitles and transcripts - subtitle_nodes = [ - ('subtitles', 'srt', 'subtitle'), - ('subtitlesTxt', 'txt', 'transcript'), - ] - for (subtitle_node, subtitle_extension, subtitle_description) in subtitle_nodes: - logging.debug('Gathering %s URLs for video_id <%s>.', subtitle_description, video_id) - subtitles = dom.get(subtitle_node) - if subtitles is not None: - if subtitle_language == 'all': - for current_subtitle_language in subtitles: - video_content[current_subtitle_language + '.' + subtitle_extension] = make_coursera_absolute_url(subtitles.get(current_subtitle_language)) - else: - if subtitle_language != 'en' and subtitle_language not in subtitles: - logging.warning("%s unavailable in '%s' language for video " - "with video id: [%s], falling back to 'en' " - "%s", subtitle_description.capitalize(), subtitle_language, video_id, subtitle_description) - subtitle_language = 'en' + subtitle_link = self._extract_subtitles_from_video_dom( + dom, subtitle_language, video_id) - subtitle_url = subtitles.get(subtitle_language) - if subtitle_url is not None: - # some subtitle urls are relative! - video_content[subtitle_language + '.' + subtitle_extension] = make_coursera_absolute_url(subtitle_url) + for key, value in iteritems(subtitle_link): + video_content[key] = value lecture_video_content = {} for key, value in iteritems(video_content): @@ -751,6 +747,102 @@ class CourseraOnDemand(object): return lecture_video_content + def _extract_subtitles_from_video_dom(self, video_dom, + subtitle_language, video_id): + # subtitles and transcripts + subtitle_nodes = [ + ('subtitles', 'srt', 'subtitle'), + ('subtitlesTxt', 'txt', 'transcript'), + ] + subtitle_set_download = set() + subtitle_set_nonexist = set() + subtitle_links = {} + for (subtitle_node, subtitle_extension, subtitle_description) \ + in subtitle_nodes: + logging.debug('Gathering %s URLs for video_id <%s>.', + subtitle_description, video_id) + subtitles = video_dom.get(subtitle_node) + download_all_subtitle = False + if subtitles is not None: + subtitles_set = set(subtitles) + requested_subtitle_list = [s.strip() for s in + subtitle_language.split(",")] + for language_with_alts in requested_subtitle_list: + if download_all_subtitle: + break + grouped_language_list = [l.strip() for l in + language_with_alts.split("|")] + for language in grouped_language_list: + if language == "all": + download_all_subtitle = True + break + elif language in subtitles_set: + subtitle_set_download.update([language]) + break + else: + subtitle_set_nonexist.update([language]) + + if download_all_subtitle and subtitles is not None: + subtitle_set_download = set(subtitles) + + if not download_all_subtitle and subtitle_set_nonexist: + logging.warning("%s unavailable in '%s' language for video " + "with video id: [%s]," + "%s", subtitle_description.capitalize(), + ", ".join(subtitle_set_nonexist), video_id, + subtitle_description) + if not subtitle_set_download: + logging.warning("%s all requested subtitles are unavaliable," + "with video id: [%s], falling back to 'en' " + "%s", subtitle_description.capitalize(), + video_id, + subtitle_description) + subtitle_set_download = set(['en']) + + for current_subtitle_language in subtitle_set_download: + subtitle_url = subtitles.get(current_subtitle_language) + if subtitle_url is not None: + # some subtitle urls are relative! + subtitle_links[ + "%s.%s" % (current_subtitle_language, subtitle_extension) + ] = make_coursera_absolute_url(subtitle_url) + return subtitle_links + + def extract_links_from_programming_immediate_instructions(self, element_id): + """ + Return a dictionary with links to supplement files (pdf, csv, zip, + ipynb, html and so on) extracted from graded programming assignment. + + @param element_id: Element ID to extract files from. + @type element_id: str + + @return: @see CourseraOnDemand._extract_links_from_text + """ + logging.debug('Extracting links from programming immediate ' + 'instructions for element_id <%s>.', element_id) + + try: + # Assignment text (instructions) contains asset tags which describe + # supplementary files. + text = ''.join( + self._extract_programming_immediate_instructions_text(element_id)) + if not text: + return {} + + supplement_links = self._extract_links_from_text(text) + instructions = (IN_MEMORY_MARKER + self._markup_to_html(text), + 'instructions') + extend_supplement_links( + supplement_links, {IN_MEMORY_EXTENSION: [instructions]}) + return supplement_links + except requests.exceptions.HTTPError as exception: + logging.error('Could not download programming assignment %s: %s', + element_id, exception) + if is_debug_run(): + logging.exception('Could not download programming assignment %s: %s', + element_id, exception) + return None + def extract_links_from_programming(self, element_id): """ Return a dictionary with links to supplement files (pdf, csv, zip, @@ -876,6 +968,87 @@ class CourseraOnDemand(object): 'url': element['url'].strip()} for element in dom['elements']] + def extract_references_poll(self): + try: + dom = get_page(self._session, + OPENCOURSE_REFERENCES_POLL_URL.format( + course_id=self._course_id), + json=True + ) + logging.info('Downloaded resource poll (%d bytes)', len(dom)) + return dom['elements'] + + except requests.exceptions.HTTPError as exception: + logging.error('Could not download resource section: %s', + exception) + if is_debug_run(): + logging.exception('Could not download resource section: %s', + exception) + return None + + def extract_links_from_reference(self, short_id): + """ + Return a dictionary with supplement files (pdf, csv, zip, ipynb, html + and so on) extracted from supplement page. + + @return: @see CourseraOnDemand._extract_links_from_text + """ + logging.debug('Gathering resource URLs for short_id <%s>.', short_id) + + try: + dom = get_page(self._session, OPENCOURSE_REFERENCE_ITEM_URL, + json=True, + course_id=self._course_id, + short_id=short_id) + + resource_content = {} + + # Supplement content has structure as follows: + # 'linked' { + # 'openCourseAssets.v1' [ { + # 'definition' { + # 'value' + + for asset in dom['linked']['openCourseAssets.v1']: + value = asset['definition']['value'] + # Supplement lecture types are known to contain both tags + # and tags (depending on the course), so we extract + # both of them. + extend_supplement_links( + resource_content, self._extract_links_from_text(value)) + + instructions = (IN_MEMORY_MARKER + self._markup_to_html(value), + 'resources') + extend_supplement_links( + resource_content, {IN_MEMORY_EXTENSION: [instructions]}) + + return resource_content + except requests.exceptions.HTTPError as exception: + logging.error('Could not download supplement %s: %s', + short_id, exception) + if is_debug_run(): + logging.exception('Could not download supplement %s: %s', + short_id, exception) + return None + + def _extract_programming_immediate_instructions_text(self, element_id): + """ + Extract assignment text (instructions). + + @param element_id: Element id to extract assignment instructions from. + @type element_id: str + + @return: List of assignment text (instructions). + @rtype: [str] + """ + dom = get_page(self._session, OPENCOURSE_PROGRAMMING_IMMEDIATE_INSTRUCTIOINS_URL, + json=True, + course_id=self._course_id, + element_id=element_id) + + return [element['assignmentInstructions']['definition']['value'] + for element in dom['elements']] + def _extract_assignment_text(self, element_id): """ Extract assignment text (instructions). diff --git a/coursera/commandline.py b/coursera/commandline.py index dc9a2bb..5b47196 100644 --- a/coursera/commandline.py +++ b/coursera/commandline.py @@ -6,13 +6,15 @@ handling. The primary candidate is argument parser. import os import sys import logging -import argparse +import configargparse as argparse from coursera import __version__ from .credentials import get_credentials, CredentialsError, keyring from .utils import decode_input +LOCAL_CONF_FILE_NAME = 'coursera-dl.conf' + def class_name_arg_required(args): """ @@ -33,8 +35,14 @@ def parse_args(args=None): Parse the arguments/options passed to the program on the command line. """ - parser = argparse.ArgumentParser( - description='Download Coursera.org lecture material and resources.') + parse_kwargs = { + "description": 'Download Coursera.org lecture material and resources.' + } + + conf_file_path = os.path.join(os.getcwd(), LOCAL_CONF_FILE_NAME) + if os.path.isfile(conf_file_path): + parse_kwargs["default_config_files"] = [conf_file_path] + parser = argparse.ArgParser(**parse_kwargs) # Basic options group_basic = parser.add_argument_group('Basic options') @@ -93,7 +101,15 @@ def parse_args(args=None): action='store', default='all', help='Choose language to download subtitles and transcripts. (Default: all)' - 'Use special value "all" to download all available.') + 'Use special value "all" to download all available.' + 'To download subtitles and transcripts of multiple languages,' + 'use comma(s) (without spaces) to seperate the names of the languages, i.e., "en,zh-CN".' + 'To download subtitles and transcripts of alternative language(s) ' + 'if only the current language is not available,' + 'put an "|" for each of the alternative languages after ' + 'the current language, i.e., "en|fr,zh-CN|zh-TW|de", and make sure the parameter are wrapped with ' + 'quotes when "|" presents.' + ) # Selection of material to download group_material = parser.add_argument_group('Selection of material to download') @@ -316,6 +332,12 @@ def parse_args(args=None): default=False, help='generate M3U playlists for course weeks') + group_adv_misc.add_argument('--mathjax-cdn', + dest='mathjax_cdn_url', + default='https://cdn.mathjax.org/mathjax/latest/MathJax.js', + help='the cdn address of MathJax.js' + ) + # Debug options group_debug = parser.add_argument_group('Debugging options') diff --git a/coursera/coursera_dl.py b/coursera/coursera_dl.py index 124868e..941793d 100644 --- a/coursera/coursera_dl.py +++ b/coursera/coursera_dl.py @@ -135,7 +135,9 @@ def download_on_demand_class(args, class_name): args.unrestricted_filenames, args.subtitle_language, args.video_resolution, - args.download_quizzes) + args.download_quizzes, + args.mathjax_cdn_url + ) if is_debug_run or args.cache_syllabus(): with open(cached_syllabus_filename, 'w') as file_object: diff --git a/coursera/define.py b/coursera/define.py index 1a72241..7b3fbe5 100644 --- a/coursera/define.py +++ b/coursera/define.py @@ -67,6 +67,12 @@ OPENCOURSE_SUPPLEMENT_URL = 'https://www.coursera.org/api/onDemandSupplements.v1 '{course_id}~{element_id}?includes=asset&fields=openCourseAssets.v1%28typeName%29,openCourseAssets.v1%28definition%29' OPENCOURSE_PROGRAMMING_ASSIGNMENTS_URL = \ 'https://www.coursera.org/api/onDemandProgrammingLearnerAssignments.v1/{course_id}~{element_id}?fields=submissionLearnerSchema' +OPENCOURSE_PROGRAMMING_IMMEDIATE_INSTRUCTIOINS_URL = \ + 'https://www.coursera.org/api/onDemandProgrammingImmediateInstructions.v1/{course_id}~{element_id}' +OPENCOURSE_REFERENCES_POLL_URL = \ + "https://www.coursera.org/api/onDemandReferences.v1/?courseId={course_id}&q=courseListed&fields=name%2CshortId%2Cslug%2Ccontent&includes=assets" +OPENCOURSE_REFERENCE_ITEM_URL = \ + "https://www.coursera.org/api/onDemandReferences.v1/?courseId={course_id}&q=shortId&shortId={short_id}&fields=name%2CshortId%2Cslug%2Ccontent&includes=assets" # These are ids that are present in tag in assignment text: # @@ -772,7 +778,7 @@ FORMAT_MAX_LENGTH = 20 TITLE_MAX_LENGTH = 200 #: CSS that is usen to prettify instructions -INSTRUCTIONS_HTML_INJECTION = ''' +INSTRUCTIONS_HTML_INJECTION_PRE = '''