Extract additional another variation of programming assignments

There are courses (e.g. data-structures, algorithms-on-graphs) that
have almost empty modules in api/opencourse.v1/course response. Such
modules look as follows:

{ // this is a module
    "id": "QgCuM", // <- lecture id
    "slug": "programming-assignment",
    "name": "Programming Assignment",
    "timeCommitment": 0,
    "elements": []
}

Their elements are empty so it's not straightforward to extract
supplementary materials (such as zip archives, PDFs) from them.
To do that, one needs to call api/onDemandCourseMaterials.v1
and parse the result. There you will see something like this:

{
  "id": "AUd0k",
  "moduleId": "0MGvs",
  "lessonId": "QgCuM",
  "name": "Programming Assignment 1: Decomposition of Graphs",
  "slug": "programming-assignment-1-decomposition-of-graphs",
  "timeCommitment": 10800000,
  "content": {
    "typeName": "gradedProgramming",
    "definition": {
      "programmingAssignmentId": "zHzR5yhHEeaE0BKOcl4zJQ@2",
      "gradingWeight": 20
    }
  },
  "isLocked": true,
  "itemLockedReasonCode": "PREMIUM",
  "trackId": "core"
},

Result is substituted into the list of lectures and is parsed
as a normal lecture.

coursera/api.py

@@ -18,9 +18,78 @@ from .define import (OPENCOURSE_SUPPLEMENT_URL,
                      OPENCOURSE_ASSET_URL,
                      OPENCOURSE_ASSETS_URL,
                      OPENCOURSE_API_ASSETS_V1_URL,
+                     OPENCOURSE_ONDEMAND_COURSE_MATERIALS,
                      OPENCOURSE_VIDEO_URL)
 
 
+class OnDemandCourseMaterialItems(object):
+    """
+    Helper class that allows accessing lecture JSONs by lesson IDs.
+    """
+    def __init__(self, items):
+        """
+        Initialization. Build a map from lessonId to Lecture (item)
+
+        @param items: linked.OnDemandCourseMaterialItems key of
+            OPENCOURSE_ONDEMAND_COURSE_MATERIALS response.
+        @type items: dict
+        """
+        # Build a map of lessonId => Item
+        self._items = dict((item['lessonId'], item) for item in items)
+
+    @staticmethod
+    def create(session, course_name):
+        """
+        Create an instance using a session and a course_name.
+
+        @param session: Requests session.
+        @type session: requests.Session
+
+        @param course_name: Course name (slug) from course json.
+        @type course_name: str
+
+        @return: Instance of OnDemandCourseMaterialItems
+        @rtype: OnDemandCourseMaterialItems
+        """
+
+        url = OPENCOURSE_ONDEMAND_COURSE_MATERIALS.format(class_name=course_name)
+        page = get_page(session, url)
+        dom = json.loads(page)
+        return OnDemandCourseMaterialItems(
+            dom['linked']['onDemandCourseMaterialItems.v1'])
+
+    def get(self, lesson_id):
+        """
+        Return lecture by lesson ID.
+
+        @param lesson_id: Lesson ID.
+        @type lesson_id: str
+
+        @return: Lesson JSON.
+        @rtype: dict
+        Example:
+        {
+          "id": "AUd0k",
+          "moduleId": "0MGvs",
+          "lessonId": "QgCuM",
+          "name": "Programming Assignment 1: Decomposition of Graphs",
+          "slug": "programming-assignment-1-decomposition-of-graphs",
+          "timeCommitment": 10800000,
+          "content": {
+            "typeName": "gradedProgramming",
+            "definition": {
+              "programmingAssignmentId": "zHzR5yhHEeaE0BKOcl4zJQ@2",
+              "gradingWeight": 20
+            }
+          },
+          "isLocked": true,
+          "itemLockedReasonCode": "PREMIUM",
+          "trackId": "core"
+        },
+        """
+        return self._items.get(lesson_id)
+
+
 class CourseraOnDemand(object):
     """
     This is a class that provides a friendly interface to extract certain

coursera/coursera_dl.py

@@ -70,8 +70,9 @@ from .define import (CLASS_URL, ABOUT_URL, PATH_CACHE,
 from .downloaders import get_downloader
 from .utils import (clean_filename, get_anchor_format, mkdir_p, fix_url,
                     decode_input, BeautifulSoup)
+
 from .network import get_page, get_page_and_url
-from .api import CourseraOnDemand
+from .api import CourseraOnDemand, OnDemandCourseMaterialItems
 from coursera import __version__
 
 # URL containing information about outdated modules
@@ -314,7 +315,9 @@ def parse_on_demand_syllabus(session, page, reverse=False, intact_fnames=False,
                  'This may take some time, be patient ...')
     modules = []
     json_modules = dom['courseMaterial']['elements']
-    course = CourseraOnDemand(session, dom['id'])
+    course = CourseraOnDemand(session=session, course_id=dom['id'])
+    ondemand_material_items = OnDemandCourseMaterialItems.create(
+        session=session, course_name=dom['slug'])
 
     for module in json_modules:
         module_slug = module['slug']
@@ -324,6 +327,15 @@ def parse_on_demand_syllabus(session, page, reverse=False, intact_fnames=False,
             section_slug = section['slug']
             lectures = []
             json_lectures = section['elements']
+
+            # Certain modules may be empty-looking programming assignments
+            # e.g. in data-structures, algorithms-on-graphs ondemand courses
+            if not json_lectures:
+                lesson_id = section['id']
+                lecture = ondemand_material_items.get(lesson_id)
+                if lecture is not None:
+                    json_lectures = [lecture]
+
             for lecture in json_lectures:
                 lecture_slug = lecture['slug']
                 typename = lecture['content']['typeName']

coursera/define.py

@@ -113,6 +113,11 @@ OPENCOURSE_ASSETS_URL = \
 OPENCOURSE_API_ASSETS_V1_URL = \
     'https://www.coursera.org/api/assets.v1/{id}'
 
+OPENCOURSE_ONDEMAND_COURSE_MATERIALS = \
+    'https://www.coursera.org/api/onDemandCourseMaterials.v1/?'\
+        'q=slug&slug={class_name}&includes=moduleIds%2ClessonIds%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2CitemIds%2Ctracks'\
+        '&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight)%2ConDemandCourseMaterialItems.v1(name%2Cslug%2CtimeCommitment%2Ccontent%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\
+        '&showLockedItems=true'
 
 ABOUT_URL = ('https://api.coursera.org/api/catalog.v1/courses?'
              'fields=largeIcon,photo,previewLink,shortDescription,smallIcon,'