diff --git a/coursera/api.py b/coursera/api.py
index 657e4b5..d1832ba 100644
--- a/coursera/api.py
+++ b/coursera/api.py
@@ -12,9 +12,11 @@ import logging
import time
import requests
import urllib
-from collections import namedtuple
+
+from collections import namedtuple, OrderedDict
from six import iterkeys, iteritems
from six.moves.urllib_parse import quote_plus
+import attr
from .utils import (BeautifulSoup, make_coursera_absolute_url,
extend_supplement_links, clean_url, clean_filename,
@@ -26,7 +28,10 @@ from .define import (OPENCOURSE_SUPPLEMENT_URL,
OPENCOURSE_ASSETS_URL,
OPENCOURSE_API_ASSETS_V1_URL,
OPENCOURSE_ONDEMAND_COURSE_MATERIALS,
- OPENCOURSE_VIDEO_URL,
+ OPENCOURSE_ONDEMAND_COURSE_MATERIALS_V2,
+ OPENCOURSE_ONDEMAND_COURSES_V1,
+ OPENCOURSE_ONDEMAND_LECTURE_VIDEOS_URL,
+ OPENCOURSE_ONDEMAND_LECTURE_ASSETS_URL,
OPENCOURSE_MEMBERSHIPS,
OPENCOURSE_REFERENCES_POLL_URL,
OPENCOURSE_REFERENCE_ITEM_URL,
@@ -278,7 +283,7 @@ class MarkupToHTMLConverter(object):
audio.insert_after(controls_tag)
-class OnDemandCourseMaterialItems(object):
+class OnDemandCourseMaterialItemsV1(object):
"""
Helper class that allows accessing lecture JSONs by lesson IDs.
"""
@@ -312,7 +317,7 @@ class OnDemandCourseMaterialItems(object):
dom = get_page(session, OPENCOURSE_ONDEMAND_COURSE_MATERIALS,
json=True,
class_name=course_name)
- return OnDemandCourseMaterialItems(
+ return OnDemandCourseMaterialItemsV1(
dom['linked']['onDemandCourseMaterialItems.v1'])
def get(self, lesson_id):
@@ -408,6 +413,134 @@ class AssetRetriever(object):
return result
+@attr.s
+class ModuleV1(object):
+ name = attr.ib()
+ id = attr.ib()
+ slug = attr.ib()
+ child_ids = attr.ib()
+
+ def children(self, all_children):
+ return [all_children[child] for child in self.child_ids]
+
+
+@attr.s
+class ModulesV1(object):
+ children = attr.ib()
+
+ @staticmethod
+ def from_json(data):
+ return ModulesV1(OrderedDict(
+ (item['id'],
+ ModuleV1(item['name'],
+ item['id'],
+ item['slug'],
+ item['lessonIds']))
+ for item in data
+ ))
+
+ def __getitem__(self, key):
+ return self.children[key]
+
+ def __iter__(self):
+ return iter(self.children.values())
+
+
+@attr.s
+class LessonV1(object):
+ name = attr.ib()
+ id = attr.ib()
+ slug = attr.ib()
+ child_ids = attr.ib()
+
+ def children(self, all_children):
+ return [all_children[child] for child in self.child_ids]
+
+
+@attr.s
+class LessonsV1(object):
+ children = attr.ib()
+
+ @staticmethod
+ def from_json(data):
+ return LessonsV1(OrderedDict(
+ (item['id'],
+ LessonV1(item['name'],
+ item['id'],
+ item['slug'],
+ item['itemIds']))
+ for item in data
+ ))
+
+ def __getitem__(self, key):
+ return self.children[key]
+
+
+@attr.s
+class ItemV2(object):
+ name = attr.ib()
+ id = attr.ib()
+ slug = attr.ib()
+ type_name = attr.ib()
+ lesson_id = attr.ib()
+ module_id = attr.ib()
+
+
+@attr.s
+class ItemsV2(object):
+ children = attr.ib()
+
+ @staticmethod
+ def from_json(data):
+ return ItemsV2({
+ item['id']:
+ ItemV2(item['name'],
+ item['id'],
+ item['slug'],
+ item['contentSummary']['typeName'],
+ item['lessonId'],
+ item['moduleId'])
+ for item in data
+ })
+
+ def __getitem__(self, key):
+ return self.children[key]
+
+
+@attr.s
+class VideoV1(object):
+ resolution = attr.ib()
+ mp4_video_url = attr.ib()
+
+
+@attr.s
+class VideosV1(object):
+ children = attr.ib()
+
+ @staticmethod
+ def from_json(data):
+
+ videos = [VideoV1(resolution, links['mp4VideoUrl'])
+ for resolution, links
+ in data['sources']['byResolution'].items()]
+ videos.sort(key=lambda video: video.resolution, reverse=True)
+
+ videos = OrderedDict(
+ (video.resolution, video)
+ for video in videos
+ )
+ return VideosV1(videos)
+
+ def __contains__(self, key):
+ return key in self.children
+
+ def __getitem__(self, key):
+ return self.children[key]
+
+ def get_best(self):
+ return next(iter(self.children.values()))
+
+
class CourseraOnDemand(object):
"""
This is a class that provides a friendly interface to extract certain
@@ -687,9 +820,9 @@ class CourseraOnDemand(object):
})
return headers
- def extract_links_from_lecture(self,
+ def extract_links_from_lecture(self, course_id,
video_id, subtitle_language='en',
- resolution='540p', assets=None):
+ resolution='540p'):
"""
Return the download URLs of on-demand course video.
@@ -702,18 +835,13 @@ class CourseraOnDemand(object):
@param resolution: Preferred video resolution.
@type resolution: str
- @param assets: List of assets that may present in the video.
- @type assets: [str]
-
@return: @see CourseraOnDemand._extract_links_from_text
"""
- if assets is None:
- assets = []
-
try:
links = self._extract_videos_and_subtitles_from_lecture(
- video_id, subtitle_language, resolution)
+ course_id, video_id, subtitle_language, resolution)
+ assets = self._get_lecture_asset_ids(course_id, video_id)
assets = self._normalize_assets(assets)
extend_supplement_links(
links, self._extract_links_from_lecture_assets(assets))
@@ -727,6 +855,17 @@ class CourseraOnDemand(object):
'Could not download lecture %s: %s', video_id, exception)
return None
+ def _get_lecture_asset_ids(self, course_id, video_id):
+ """
+ Obtain a list of asset ids from a lecture.
+ """
+ dom = get_page(self._session, OPENCOURSE_ONDEMAND_LECTURE_ASSETS_URL,
+ json=True, course_id=course_id, video_id=video_id)
+ # Note that we extract here "id", not definition -> assetId, as it
+ # be extracted later.
+ return [asset['id']
+ for asset in dom['linked']['openCourseAssets.v1']]
+
def _normalize_assets(self, assets):
"""
Perform asset normalization. For some reason, assets that are sometimes
@@ -850,41 +989,34 @@ class CourseraOnDemand(object):
return urls
def _extract_videos_and_subtitles_from_lecture(self,
+ course_id,
video_id,
subtitle_language='en',
resolution='540p'):
- dom = get_page(self._session, OPENCOURSE_VIDEO_URL,
- json=True,
- video_id=video_id)
-
logging.debug('Parsing JSON for video_id <%s>.', video_id)
+
+ dom = get_page(self._session, OPENCOURSE_ONDEMAND_LECTURE_VIDEOS_URL,
+ json=True,
+ course_id=course_id,
+ video_id=video_id)
+ dom = dom['linked']['onDemandVideos.v1'][0]
+
+ videos = VideosV1.from_json(dom)
video_content = {}
- # videos
- logging.debug('Gathering video URLs for video_id <%s>.', video_id)
- sources = dom['sources']
- sources.sort(key=lambda src: src['resolution'])
- sources.reverse()
-
- # Try to select resolution requested by the user.
- filtered_sources = [source
- for source in sources
- if source['resolution'] == resolution]
-
- if len(filtered_sources) == 0:
- # We will just use the 'vanilla' version of sources here, instead of
- # filtered_sources.
- logging.warning('Requested resolution %s not available for <%s>. '
- 'Downloading highest resolution available instead.',
- resolution, video_id)
- else:
+ if resolution in videos:
+ source = videos[resolution]
logging.debug('Proceeding with download of resolution %s of <%s>.',
resolution, video_id)
- sources = filtered_sources
+ else:
+ source = videos.get_best()
+ logging.warning(
+ 'Requested resolution %s not available for <%s>. '
+ 'Downloading highest resolution (%s) available instead.',
+ resolution, video_id, source.resolution)
- video_url = sources[0]['formatSources']['video/mp4']
- video_content['mp4'] = video_url
+ video_content['mp4'] = source.mp4_video_url
subtitle_link = self._extract_subtitles_from_video_dom(
dom, subtitle_language, video_id)
diff --git a/coursera/coursera_dl.py b/coursera/coursera_dl.py
index 4c20689..a389106 100644
--- a/coursera/coursera_dl.py
+++ b/coursera/coursera_dl.py
@@ -67,7 +67,8 @@ from .workflow import CourseraDownloader
from .parallel import ConsecutiveDownloader, ParallelDownloader
from .utils import (clean_filename, get_anchor_format, mkdir_p, fix_url,
print_ssl_error_message,
- decode_input, BeautifulSoup, is_debug_run)
+ decode_input, BeautifulSoup, is_debug_run,
+ spit_json, slurp_json)
from .network import get_page, get_page_and_url
from .commandline import parse_args
@@ -126,8 +127,7 @@ def download_on_demand_class(args, class_name):
cached_syllabus_filename = '%s-syllabus-parsed.json' % class_name
if args.cache_syllabus and os.path.isfile(cached_syllabus_filename):
- with open(cached_syllabus_filename) as syllabus_file:
- modules = json.load(syllabus_file)
+ modules = slurp_json(cached_syllabus_filename)
else:
error_occured, modules = extractor.get_modules(
class_name,
@@ -141,8 +141,7 @@ def download_on_demand_class(args, class_name):
)
if is_debug_run or args.cache_syllabus():
- with open(cached_syllabus_filename, 'w') as file_object:
- json.dump(modules, file_object, indent=4)
+ spit_json(modules, cached_syllabus_filename)
if args.only_syllabus:
return error_occured, False
diff --git a/coursera/define.py b/coursera/define.py
index e2ec105..ff8c51e 100644
--- a/coursera/define.py
+++ b/coursera/define.py
@@ -61,8 +61,10 @@ OPENCOURSE_LIST_COURSES = 'https://api.coursera.org/api/courses.v1?q=watchlist&s
# }
# }
OPENCOURSE_MEMBERSHIPS = 'https://api.coursera.org/api/memberships.v1?includes=courseId,courses.v1&q=me&showHidden=true&filter=current,preEnrolled'
-OPENCOURSE_CONTENT_URL = 'https://api.coursera.org/api/opencourse.v1/course/{class_name}?showLockedItems=true'
-OPENCOURSE_VIDEO_URL = 'https://api.coursera.org/api/opencourse.v1/video/{video_id}'
+OPENCOURSE_ONDEMAND_LECTURE_VIDEOS_URL = \
+ 'https://api.coursera.org/api/onDemandLectureVideos.v1/'\
+ '{course_id}~{video_id}?includes=video&'\
+ 'fields=onDemandVideos.v1(sources%2Csubtitles%2CsubtitlesVtt%2CsubtitlesTxt)'
OPENCOURSE_SUPPLEMENT_URL = 'https://api.coursera.org/api/onDemandSupplements.v1/'\
'{course_id}~{element_id}?includes=asset&fields=openCourseAssets.v1%28typeName%29,openCourseAssets.v1%28definition%29'
OPENCOURSE_PROGRAMMING_ASSIGNMENTS_URL = \
@@ -97,6 +99,23 @@ OPENCOURSE_REFERENCE_ITEM_URL = \
OPENCOURSE_ASSET_URL = \
'https://api.coursera.org/api/assetUrls.v1?ids={ids}'
+# Sample response:
+# "linked": {
+# "openCourseAssets.v1": [
+# {
+# "typeName": "asset",
+# "definition": {
+# "assetId": "fytYX5rYEeedWRLokafKRg",
+# "name": "Lecture slides"
+# },
+# "id": "j6g7VZrYEeeUVgpv-dYMig"
+# }
+# ]
+# }
+OPENCOURSE_ONDEMAND_LECTURE_ASSETS_URL = \
+ 'https://api.coursera.org/api/onDemandLectureAssets.v1/'\
+ '{course_id}~{video_id}/?includes=openCourseAssets'
+
# These ids are provided in lecture json:
#
# {
@@ -170,9 +189,20 @@ OPENCOURSE_API_ASSETS_V1_URL = \
OPENCOURSE_ONDEMAND_COURSE_MATERIALS = \
'https://api.coursera.org/api/onDemandCourseMaterials.v1/?'\
- 'q=slug&slug={class_name}&includes=moduleIds%2ClessonIds%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2CitemIds%2Ctracks'\
- '&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight)%2ConDemandCourseMaterialItems.v1(name%2Cslug%2CtimeCommitment%2Ccontent%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\
- '&showLockedItems=true'
+ 'q=slug&slug={class_name}&includes=moduleIds%2ClessonIds%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2CitemIds%2Ctracks'\
+ '&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight)%2ConDemandCourseMaterialItems.v1(name%2Cslug%2CtimeCommitment%2Ccontent%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\
+ '&showLockedItems=true'
+
+OPENCOURSE_ONDEMAND_COURSE_MATERIALS_V2 = \
+ 'https://api.coursera.org/api/onDemandCourseMaterials.v2/?q=slug&slug={class_name}'\
+ '&includes=modules%2Clessons%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2Citems%2Ctracks%2CgradePolicy&'\
+ '&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional%2ClearningObjectives)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight%2CisRequiredForPassing)%2ConDemandCourseMaterialItems.v2(name%2Cslug%2CtimeCommitment%2CcontentSummary%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId%2ClockedStatus%2CitemLockSummary)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\
+ '&showLockedItems=true'
+
+OPENCOURSE_ONDEMAND_COURSES_V1 = \
+ 'https://api.coursera.org/api/onDemandCourses.v1?q=slug&slug={class_name}&'\
+ 'includes=instructorIds%2CpartnerIds%2C_links&'\
+ 'fields=brandingImage%2CcertificatePurchaseEnabledAt%2Cpartners.v1(squareLogo%2CrectangularLogo)%2Cinstructors.v1(fullName)%2CoverridePartnerLogos%2CsessionsEnabledAt%2CdomainTypes%2CpremiumExperienceVariant%2CisRestrictedMembership'
ABOUT_URL = ('https://api.coursera.org/api/catalog.v1/courses?'
'fields=largeIcon,photo,previewLink,shortDescription,smallIcon,'
@@ -924,7 +954,7 @@ pre {