Move to newer API for syllabus and lecture retrieval

ref #665
ref #673
ref #634

coursera/api.py

@@ -12,9 +12,11 @@ import logging
 import time
 import requests
 import urllib
-from collections import namedtuple
+
+from collections import namedtuple, OrderedDict
 from six import iterkeys, iteritems
 from six.moves.urllib_parse import quote_plus
+import attr
 
 from .utils import (BeautifulSoup, make_coursera_absolute_url,
                     extend_supplement_links, clean_url, clean_filename,
@@ -26,7 +28,10 @@ from .define import (OPENCOURSE_SUPPLEMENT_URL,
                      OPENCOURSE_ASSETS_URL,
                      OPENCOURSE_API_ASSETS_V1_URL,
                      OPENCOURSE_ONDEMAND_COURSE_MATERIALS,
-                     OPENCOURSE_VIDEO_URL,
+                     OPENCOURSE_ONDEMAND_COURSE_MATERIALS_V2,
+                     OPENCOURSE_ONDEMAND_COURSES_V1,
+                     OPENCOURSE_ONDEMAND_LECTURE_VIDEOS_URL,
+                     OPENCOURSE_ONDEMAND_LECTURE_ASSETS_URL,
                      OPENCOURSE_MEMBERSHIPS,
                      OPENCOURSE_REFERENCES_POLL_URL,
                      OPENCOURSE_REFERENCE_ITEM_URL,
@@ -278,7 +283,7 @@ class MarkupToHTMLConverter(object):
                 audio.insert_after(controls_tag)
 
 
-class OnDemandCourseMaterialItems(object):
+class OnDemandCourseMaterialItemsV1(object):
     """
     Helper class that allows accessing lecture JSONs by lesson IDs.
     """
@@ -312,7 +317,7 @@ class OnDemandCourseMaterialItems(object):
         dom = get_page(session, OPENCOURSE_ONDEMAND_COURSE_MATERIALS,
                        json=True,
                        class_name=course_name)
-        return OnDemandCourseMaterialItems(
+        return OnDemandCourseMaterialItemsV1(
             dom['linked']['onDemandCourseMaterialItems.v1'])
 
     def get(self, lesson_id):
@@ -408,6 +413,134 @@ class AssetRetriever(object):
         return result
 
 
+@attr.s
+class ModuleV1(object):
+    name = attr.ib()
+    id = attr.ib()
+    slug = attr.ib()
+    child_ids = attr.ib()
+
+    def children(self, all_children):
+        return [all_children[child] for child in self.child_ids]
+
+
+@attr.s
+class ModulesV1(object):
+    children = attr.ib()
+
+    @staticmethod
+    def from_json(data):
+        return ModulesV1(OrderedDict(
+            (item['id'],
+             ModuleV1(item['name'],
+                      item['id'],
+                      item['slug'],
+                      item['lessonIds']))
+            for item in data
+        ))
+
+    def __getitem__(self, key):
+        return self.children[key]
+
+    def __iter__(self):
+        return iter(self.children.values())
+
+
+@attr.s
+class LessonV1(object):
+    name = attr.ib()
+    id = attr.ib()
+    slug = attr.ib()
+    child_ids = attr.ib()
+
+    def children(self, all_children):
+        return [all_children[child] for child in self.child_ids]
+
+
+@attr.s
+class LessonsV1(object):
+    children = attr.ib()
+
+    @staticmethod
+    def from_json(data):
+        return LessonsV1(OrderedDict(
+            (item['id'],
+             LessonV1(item['name'],
+                      item['id'],
+                      item['slug'],
+                      item['itemIds']))
+            for item in data
+        ))
+
+    def __getitem__(self, key):
+        return self.children[key]
+
+
+@attr.s
+class ItemV2(object):
+    name = attr.ib()
+    id = attr.ib()
+    slug = attr.ib()
+    type_name = attr.ib()
+    lesson_id = attr.ib()
+    module_id = attr.ib()
+
+
+@attr.s
+class ItemsV2(object):
+    children = attr.ib()
+
+    @staticmethod
+    def from_json(data):
+        return ItemsV2({
+            item['id']:
+            ItemV2(item['name'],
+                   item['id'],
+                   item['slug'],
+                   item['contentSummary']['typeName'],
+                   item['lessonId'],
+                   item['moduleId'])
+            for item in data
+        })
+
+    def __getitem__(self, key):
+        return self.children[key]
+
+
+@attr.s
+class VideoV1(object):
+    resolution = attr.ib()
+    mp4_video_url = attr.ib()
+
+
+@attr.s
+class VideosV1(object):
+    children = attr.ib()
+
+    @staticmethod
+    def from_json(data):
+
+        videos = [VideoV1(resolution, links['mp4VideoUrl'])
+                  for resolution, links
+                  in data['sources']['byResolution'].items()]
+        videos.sort(key=lambda video: video.resolution, reverse=True)
+
+        videos = OrderedDict(
+            (video.resolution, video)
+            for video in videos
+        )
+        return VideosV1(videos)
+
+    def __contains__(self, key):
+        return key in self.children
+
+    def __getitem__(self, key):
+        return self.children[key]
+
+    def get_best(self):
+        return next(iter(self.children.values()))
+
+
 class CourseraOnDemand(object):
     """
     This is a class that provides a friendly interface to extract certain
@@ -687,9 +820,9 @@ class CourseraOnDemand(object):
         })
         return headers
 
-    def extract_links_from_lecture(self,
+    def extract_links_from_lecture(self, course_id,
                                    video_id, subtitle_language='en',
-                                   resolution='540p', assets=None):
+                                   resolution='540p'):
         """
         Return the download URLs of on-demand course video.
 
@@ -702,18 +835,13 @@ class CourseraOnDemand(object):
         @param resolution: Preferred video resolution.
         @type resolution: str
 
-        @param assets: List of assets that may present in the video.
-        @type assets: [str]
-
         @return: @see CourseraOnDemand._extract_links_from_text
         """
-        if assets is None:
-            assets = []
-
         try:
             links = self._extract_videos_and_subtitles_from_lecture(
-                video_id, subtitle_language, resolution)
+                course_id, video_id, subtitle_language, resolution)
 
+            assets = self._get_lecture_asset_ids(course_id, video_id)
             assets = self._normalize_assets(assets)
             extend_supplement_links(
                 links, self._extract_links_from_lecture_assets(assets))
@@ -727,6 +855,17 @@ class CourseraOnDemand(object):
                     'Could not download lecture %s: %s', video_id, exception)
             return None
 
+    def _get_lecture_asset_ids(self, course_id, video_id):
+        """
+        Obtain a list of asset ids from a lecture.
+        """
+        dom = get_page(self._session, OPENCOURSE_ONDEMAND_LECTURE_ASSETS_URL,
+                       json=True, course_id=course_id, video_id=video_id)
+        # Note that we extract here "id", not definition -> assetId, as it
+        # be extracted later.
+        return [asset['id']
+                for asset in dom['linked']['openCourseAssets.v1']]
+
     def _normalize_assets(self, assets):
         """
         Perform asset normalization. For some reason, assets that are sometimes
@@ -850,41 +989,34 @@ class CourseraOnDemand(object):
         return urls
 
     def _extract_videos_and_subtitles_from_lecture(self,
+                                                   course_id,
                                                    video_id,
                                                    subtitle_language='en',
                                                    resolution='540p'):
 
-        dom = get_page(self._session, OPENCOURSE_VIDEO_URL,
-                       json=True,
-                       video_id=video_id)
-
         logging.debug('Parsing JSON for video_id <%s>.', video_id)
+
+        dom = get_page(self._session, OPENCOURSE_ONDEMAND_LECTURE_VIDEOS_URL,
+                       json=True,
+                       course_id=course_id,
+                       video_id=video_id)
+        dom = dom['linked']['onDemandVideos.v1'][0]
+
+        videos = VideosV1.from_json(dom)
         video_content = {}
 
-        # videos
-        logging.debug('Gathering video URLs for video_id <%s>.', video_id)
-        sources = dom['sources']
-        sources.sort(key=lambda src: src['resolution'])
-        sources.reverse()
-
-        # Try to select resolution requested by the user.
-        filtered_sources = [source
-                            for source in sources
-                            if source['resolution'] == resolution]
-
-        if len(filtered_sources) == 0:
-            # We will just use the 'vanilla' version of sources here, instead of
-            # filtered_sources.
-            logging.warning('Requested resolution %s not available for <%s>. '
-                            'Downloading highest resolution available instead.',
-                            resolution, video_id)
-        else:
+        if resolution in videos:
+            source = videos[resolution]
             logging.debug('Proceeding with download of resolution %s of <%s>.',
                           resolution, video_id)
-            sources = filtered_sources
+        else:
+            source = videos.get_best()
+            logging.warning(
+                'Requested resolution %s not available for <%s>. '
+                'Downloading highest resolution (%s) available instead.',
+                resolution, video_id, source.resolution)
 
-        video_url = sources[0]['formatSources']['video/mp4']
-        video_content['mp4'] = video_url
+        video_content['mp4'] = source.mp4_video_url
 
         subtitle_link = self._extract_subtitles_from_video_dom(
             dom, subtitle_language, video_id)

coursera/coursera_dl.py

@@ -67,7 +67,8 @@ from .workflow import CourseraDownloader
 from .parallel import ConsecutiveDownloader, ParallelDownloader
 from .utils import (clean_filename, get_anchor_format, mkdir_p, fix_url,
                     print_ssl_error_message,
-                    decode_input, BeautifulSoup, is_debug_run)
+                    decode_input, BeautifulSoup, is_debug_run,
+                    spit_json, slurp_json)
 
 from .network import get_page, get_page_and_url
 from .commandline import parse_args
@@ -126,8 +127,7 @@ def download_on_demand_class(args, class_name):
 
     cached_syllabus_filename = '%s-syllabus-parsed.json' % class_name
     if args.cache_syllabus and os.path.isfile(cached_syllabus_filename):
-        with open(cached_syllabus_filename) as syllabus_file:
-            modules = json.load(syllabus_file)
+        modules = slurp_json(cached_syllabus_filename)
     else:
         error_occured, modules = extractor.get_modules(
             class_name,
@@ -141,8 +141,7 @@ def download_on_demand_class(args, class_name):
         )
 
     if is_debug_run or args.cache_syllabus():
-        with open(cached_syllabus_filename, 'w') as file_object:
-            json.dump(modules, file_object, indent=4)
+        spit_json(modules, cached_syllabus_filename)
 
     if args.only_syllabus:
         return error_occured, False

coursera/define.py

@@ -61,8 +61,10 @@ OPENCOURSE_LIST_COURSES = 'https://api.coursera.org/api/courses.v1?q=watchlist&s
 #     }
 # }
 OPENCOURSE_MEMBERSHIPS = 'https://api.coursera.org/api/memberships.v1?includes=courseId,courses.v1&q=me&showHidden=true&filter=current,preEnrolled'
-OPENCOURSE_CONTENT_URL = 'https://api.coursera.org/api/opencourse.v1/course/{class_name}?showLockedItems=true'
-OPENCOURSE_VIDEO_URL = 'https://api.coursera.org/api/opencourse.v1/video/{video_id}'
+OPENCOURSE_ONDEMAND_LECTURE_VIDEOS_URL = \
+    'https://api.coursera.org/api/onDemandLectureVideos.v1/'\
+    '{course_id}~{video_id}?includes=video&'\
+    'fields=onDemandVideos.v1(sources%2Csubtitles%2CsubtitlesVtt%2CsubtitlesTxt)'
 OPENCOURSE_SUPPLEMENT_URL = 'https://api.coursera.org/api/onDemandSupplements.v1/'\
     '{course_id}~{element_id}?includes=asset&fields=openCourseAssets.v1%28typeName%29,openCourseAssets.v1%28definition%29'
 OPENCOURSE_PROGRAMMING_ASSIGNMENTS_URL = \
@@ -97,6 +99,23 @@ OPENCOURSE_REFERENCE_ITEM_URL = \
 OPENCOURSE_ASSET_URL = \
     'https://api.coursera.org/api/assetUrls.v1?ids={ids}'
 
+# Sample response:
+#  "linked": {
+#    "openCourseAssets.v1": [
+#      {
+#        "typeName": "asset",
+#        "definition": {
+#          "assetId": "fytYX5rYEeedWRLokafKRg",
+#          "name": "Lecture slides"
+#        },
+#        "id": "j6g7VZrYEeeUVgpv-dYMig"
+#      }
+#    ]
+#  }
+OPENCOURSE_ONDEMAND_LECTURE_ASSETS_URL = \
+    'https://api.coursera.org/api/onDemandLectureAssets.v1/'\
+    '{course_id}~{video_id}/?includes=openCourseAssets'
+
 # These ids are provided in lecture json:
 #
 # {
@@ -170,9 +189,20 @@ OPENCOURSE_API_ASSETS_V1_URL = \
 
 OPENCOURSE_ONDEMAND_COURSE_MATERIALS = \
     'https://api.coursera.org/api/onDemandCourseMaterials.v1/?'\
-        'q=slug&slug={class_name}&includes=moduleIds%2ClessonIds%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2CitemIds%2Ctracks'\
-        '&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight)%2ConDemandCourseMaterialItems.v1(name%2Cslug%2CtimeCommitment%2Ccontent%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\
-        '&showLockedItems=true'
+    'q=slug&slug={class_name}&includes=moduleIds%2ClessonIds%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2CitemIds%2Ctracks'\
+    '&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight)%2ConDemandCourseMaterialItems.v1(name%2Cslug%2CtimeCommitment%2Ccontent%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\
+    '&showLockedItems=true'
+
+OPENCOURSE_ONDEMAND_COURSE_MATERIALS_V2 = \
+    'https://api.coursera.org/api/onDemandCourseMaterials.v2/?q=slug&slug={class_name}'\
+    '&includes=modules%2Clessons%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2Citems%2Ctracks%2CgradePolicy&'\
+    '&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional%2ClearningObjectives)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight%2CisRequiredForPassing)%2ConDemandCourseMaterialItems.v2(name%2Cslug%2CtimeCommitment%2CcontentSummary%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId%2ClockedStatus%2CitemLockSummary)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\
+    '&showLockedItems=true'
+
+OPENCOURSE_ONDEMAND_COURSES_V1 = \
+    'https://api.coursera.org/api/onDemandCourses.v1?q=slug&slug={class_name}&'\
+    'includes=instructorIds%2CpartnerIds%2C_links&'\
+    'fields=brandingImage%2CcertificatePurchaseEnabledAt%2Cpartners.v1(squareLogo%2CrectangularLogo)%2Cinstructors.v1(fullName)%2CoverridePartnerLogos%2CsessionsEnabledAt%2CdomainTypes%2CpremiumExperienceVariant%2CisRestrictedMembership'
 
 ABOUT_URL = ('https://api.coursera.org/api/catalog.v1/courses?'
              'fields=largeIcon,photo,previewLink,shortDescription,smallIcon,'
@@ -924,7 +954,7 @@ pre {
 <script type="text/javascript" async
   src="'''
 INSTRUCTIONS_HTML_MATHJAX_URL = 'https://cdn.mathjax.org/mathjax/latest/MathJax.js'
-INSTRUCTIONS_HTML_INJECTION_AFTER ='''?config=TeX-AMS-MML_HTMLorMML">
+INSTRUCTIONS_HTML_INJECTION_AFTER = '''?config=TeX-AMS-MML_HTMLorMML">
 </script>
 
 <script type="text/x-mathjax-config">

coursera/extractors.py

@@ -9,11 +9,12 @@ import abc
 import json
 import logging
 
-from .api import CourseraOnDemand, OnDemandCourseMaterialItems
-from .define import OPENCOURSE_CONTENT_URL
+from .api import (CourseraOnDemand, OnDemandCourseMaterialItemsV1,
+                  ModulesV1, LessonsV1, ItemsV2)
+from .define import OPENCOURSE_ONDEMAND_COURSE_MATERIALS_V2
 from .cookies import login
 from .network import get_page
-from .utils import is_debug_run
+from .utils import is_debug_run, spit_json, slurp_json
 
 
 class PlatformExtractor(object):
@@ -52,9 +53,11 @@ class CourseraExtractor(PlatformExtractor):
 
         page = self._get_on_demand_syllabus(class_name)
         error_occured, modules = self._parse_on_demand_syllabus(
+            class_name,
             page, reverse, unrestricted_filenames,
             subtitle_language, video_resolution,
             download_quizzes, mathjax_cdn_url, download_notebooks)
+
         return error_occured, modules
 
     def _get_on_demand_syllabus(self, class_name):
@@ -62,13 +65,14 @@ class CourseraExtractor(PlatformExtractor):
         Get the on-demand course listing webpage.
         """
 
-        url = OPENCOURSE_CONTENT_URL.format(class_name=class_name)
+        url = OPENCOURSE_ONDEMAND_COURSE_MATERIALS_V2.format(
+            class_name=class_name)
         page = get_page(self._session, url)
-        logging.info('Downloaded %s (%d bytes)', url, len(page))
+        logging.debug('Downloaded %s (%d bytes)', url, len(page))
 
         return page
 
-    def _parse_on_demand_syllabus(self, page, reverse=False,
+    def _parse_on_demand_syllabus(self, course_name, page, reverse=False,
                                   unrestricted_filenames=False,
                                   subtitle_language='en',
                                   video_resolution=None,
@@ -86,115 +90,132 @@ class CourseraExtractor(PlatformExtractor):
         """
 
         dom = json.loads(page)
-        course_name = dom['slug']
+        class_id = dom['elements'][0]['id']
 
-        logging.info('Parsing syllabus of on-demand course. '
-                     'This may take some time, please be patient ...')
+        logging.info('Parsing syllabus of on-demand course (id=%s). '
+                     'This may take some time, please be patient ...',
+                     class_id)
         modules = []
-        json_modules = dom['courseMaterial']['elements']
-        course = CourseraOnDemand(session=self._session, course_id=dom['id'],
-                                  course_name=course_name,
-                                  unrestricted_filenames=unrestricted_filenames,
-                                  mathjax_cdn_url=mathjax_cdn_url
-                                  )
+
+        json_modules = dom['linked']['onDemandCourseMaterialItems.v2']
+        course = CourseraOnDemand(
+            session=self._session, course_id=class_id,
+            course_name=course_name,
+            unrestricted_filenames=unrestricted_filenames,
+            mathjax_cdn_url=mathjax_cdn_url)
         course.obtain_user_id()
-        ondemand_material_items = OnDemandCourseMaterialItems.create(
+        ondemand_material_items = OnDemandCourseMaterialItemsV1.create(
             session=self._session, course_name=course_name)
 
         if is_debug_run():
-            with open('%s-syllabus-raw.json' % course_name, 'w') as file_object:
-                json.dump(dom, file_object, indent=4)
-            with open('%s-course-material-items.json' % course_name, 'w') as file_object:
-                json.dump(ondemand_material_items._items, file_object, indent=4)
+            spit_json(dom, '%s-syllabus-raw.json' % course_name)
+            spit_json(json_modules, '%s-material-items-v2.json' % course_name)
+            spit_json(ondemand_material_items._items,
+                      '%s-course-material-items.json' % course_name)
 
         error_occured = False
 
-        for module in json_modules:
-            module_slug = module['slug']
-            logging.info('Processing module  %s', module_slug)
-            sections = []
-            json_sections = module['elements']
-            for section in json_sections:
-                section_slug = section['slug']
-                logging.info('Processing section     %s', section_slug)
+        all_modules = ModulesV1.from_json(
+            dom['linked']['onDemandCourseMaterialModules.v1'])
+        all_lessons = LessonsV1.from_json(
+            dom['linked']['onDemandCourseMaterialLessons.v1'])
+        all_items = ItemsV2.from_json(
+            dom['linked']['onDemandCourseMaterialItems.v2'])
+
+        for module in all_modules:
+            logging.info('Processing module  %s', module.slug)
+            lessons = []
+            for section in module.children(all_lessons):
+                logging.info('Processing section     %s', section.slug)
                 lectures = []
-                json_lectures = section['elements']
+                available_lectures = section.children(all_items)
 
                 # Certain modules may be empty-looking programming assignments
-                # e.g. in data-structures, algorithms-on-graphs ondemand courses
-                if not json_lectures:
-                    lesson_id = section['id']
-                    lecture = ondemand_material_items.get(lesson_id)
+                # e.g. in data-structures, algorithms-on-graphs ondemand
+                # courses
+                if not available_lectures:
+                    lecture = ondemand_material_items.get(section.id)
                     if lecture is not None:
-                        json_lectures = [lecture]
+                        available_lectures = [lecture]
 
-                for lecture in json_lectures:
-                    lecture_slug = lecture['slug']
-                    typename = lecture['content']['typeName']
+                for lecture in available_lectures:
+                    typename = lecture.type_name
 
                     logging.info('Processing lecture         %s (%s)',
-                                 lecture_slug, typename)
+                                 lecture.slug, typename)
                     # Empty dictionary means there were no data
                     # None means an error occured
                     links = {}
 
                     if typename == 'lecture':
-                        lecture_video_id = lecture['content']['definition']['videoId']
-                        assets = lecture['content']['definition'].get('assets', [])
+                        # lecture_video_id = lecture['content']['definition']['videoId']
+                        # assets = lecture['content']['definition'].get(
+                        #     'assets', [])
+                        lecture_video_id = lecture.id
+                        # assets = []
 
                         links = course.extract_links_from_lecture(
+                            class_id,
                             lecture_video_id, subtitle_language,
-                            video_resolution, assets)
+                            video_resolution)
 
                     elif typename == 'supplement':
-                        links = course.extract_links_from_supplement(lecture['id'])
+                        links = course.extract_links_from_supplement(
+                            lecture.id)
 
                     elif typename == 'phasedPeer':
-                        links = course.extract_links_from_peer_assignment(lecture['id'])
+                        links = course.extract_links_from_peer_assignment(
+                            lecture.id)
 
                     elif typename in ('gradedProgramming', 'ungradedProgramming'):
-                        links = course.extract_links_from_programming(lecture['id'])
+                        links = course.extract_links_from_programming(
+                            lecture.id)
 
                     elif typename == 'quiz':
                         if download_quizzes:
-                            links = course.extract_links_from_quiz(lecture['id'])
+                            links = course.extract_links_from_quiz(
+                                lecture.id)
 
                     elif typename == 'exam':
                         if download_quizzes:
-                            links = course.extract_links_from_exam(lecture['id'])
+                            links = course.extract_links_from_exam(
+                                lecture.id)
 
                     elif typename == 'programming':
                         if download_quizzes:
-                            links = course.extract_links_from_programming_immediate_instructions(lecture['id'])
+                            links = course.extract_links_from_programming_immediate_instructions(
+                                lecture.id)
 
                     elif typename == 'notebook':
-                        if download_notebooks and self._notebook_downloaded == False:
-                            logging.warning('According to notebooks platform, content will be downloaded first')
-                            links = course.extract_links_from_notebook(lecture['id'])
+                        if download_notebooks and not self._notebook_downloaded:
+                            logging.warning(
+                                'According to notebooks platform, content will be downloaded first')
+                            links = course.extract_links_from_notebook(
+                                lecture.id)
                             self._notebook_downloaded = True
 
                     else:
                         logging.info(
                             'Unsupported typename "%s" in lecture "%s" (lecture id "%s")',
-                            typename, lecture_slug, lecture['id'])
+                            typename, lecture.slug, lecture.id)
                         continue
 
                     if links is None:
                         error_occured = True
                     elif links:
-                        lectures.append((lecture_slug, links))
+                        lectures.append((lecture.slug, links))
 
                 if lectures:
-                    sections.append((section_slug, lectures))
+                    lessons.append((section.slug, lectures))
 
-            if sections:
-                modules.append((module_slug, sections))
+            if lessons:
+                modules.append((module.slug, lessons))
 
         if modules and reverse:
             modules.reverse()
 
         # Processing resources section
-        json_references= course.extract_references_poll()
+        json_references = course.extract_references_poll()
         references = []
         if json_references:
             logging.info('Processing resources')
@@ -204,7 +225,8 @@ class CourseraExtractor(PlatformExtractor):
                 logging.info('Processing resource  %s',
                              reference_slug)
 
-                links = course.extract_links_from_reference(json_reference['shortId'])
+                links = course.extract_links_from_reference(
+                    json_reference['shortId'])
                 if links is None:
                     error_occured = True
                 elif links:

coursera/test/test_api.py

@@ -73,7 +73,7 @@ def test_extract_links_from_lecture_http_error(get_page, course):
     locked_response.status_code = define.HTTP_FORBIDDEN
     get_page.side_effect = HTTPError('Mocked HTTP error',
                                      response=locked_response)
-    assert None == course.extract_links_from_lecture('0')
+    assert None == course.extract_links_from_lecture('fake_course_id', '0')
 
 
 @patch('coursera.api.get_page')

coursera/utils.py

@@ -8,6 +8,7 @@ import os
 import re
 import sys
 import time
+import json
 import errno
 import random
 import string
@@ -41,7 +42,9 @@ else:
 from .define import COURSERA_URL, WINDOWS_UNC_PREFIX
 
 # Force us of bs4 with html.parser
-BeautifulSoup = lambda page: BeautifulSoup_(page, 'html.parser')
+
+
+def BeautifulSoup(page): return BeautifulSoup_(page, 'html.parser')
 
 
 if six.PY2:
@@ -55,6 +58,16 @@ else:
         return x
 
 
+def spit_json(obj, filename):
+    with open(filename, 'w') as file_object:
+        json.dump(obj, file_object, indent=4)
+
+
+def slurp_json(filename):
+    with open(filename) as file_object:
+        return json.load(file_object)
+
+
 def is_debug_run():
     """
     Check whether we're running with DEBUG loglevel.

requirements.txt

@@ -5,3 +5,4 @@ urllib3>=1.10
 pyasn1>=0.1.7
 keyring>=4.0
 configargparse>=0.12.0
+attrs==18.1.0