diff --git a/coursera/api.py b/coursera/api.py index 657e4b5..d1832ba 100644 --- a/coursera/api.py +++ b/coursera/api.py @@ -12,9 +12,11 @@ import logging import time import requests import urllib -from collections import namedtuple + +from collections import namedtuple, OrderedDict from six import iterkeys, iteritems from six.moves.urllib_parse import quote_plus +import attr from .utils import (BeautifulSoup, make_coursera_absolute_url, extend_supplement_links, clean_url, clean_filename, @@ -26,7 +28,10 @@ from .define import (OPENCOURSE_SUPPLEMENT_URL, OPENCOURSE_ASSETS_URL, OPENCOURSE_API_ASSETS_V1_URL, OPENCOURSE_ONDEMAND_COURSE_MATERIALS, - OPENCOURSE_VIDEO_URL, + OPENCOURSE_ONDEMAND_COURSE_MATERIALS_V2, + OPENCOURSE_ONDEMAND_COURSES_V1, + OPENCOURSE_ONDEMAND_LECTURE_VIDEOS_URL, + OPENCOURSE_ONDEMAND_LECTURE_ASSETS_URL, OPENCOURSE_MEMBERSHIPS, OPENCOURSE_REFERENCES_POLL_URL, OPENCOURSE_REFERENCE_ITEM_URL, @@ -278,7 +283,7 @@ class MarkupToHTMLConverter(object): audio.insert_after(controls_tag) -class OnDemandCourseMaterialItems(object): +class OnDemandCourseMaterialItemsV1(object): """ Helper class that allows accessing lecture JSONs by lesson IDs. """ @@ -312,7 +317,7 @@ class OnDemandCourseMaterialItems(object): dom = get_page(session, OPENCOURSE_ONDEMAND_COURSE_MATERIALS, json=True, class_name=course_name) - return OnDemandCourseMaterialItems( + return OnDemandCourseMaterialItemsV1( dom['linked']['onDemandCourseMaterialItems.v1']) def get(self, lesson_id): @@ -408,6 +413,134 @@ class AssetRetriever(object): return result +@attr.s +class ModuleV1(object): + name = attr.ib() + id = attr.ib() + slug = attr.ib() + child_ids = attr.ib() + + def children(self, all_children): + return [all_children[child] for child in self.child_ids] + + +@attr.s +class ModulesV1(object): + children = attr.ib() + + @staticmethod + def from_json(data): + return ModulesV1(OrderedDict( + (item['id'], + ModuleV1(item['name'], + item['id'], + item['slug'], + item['lessonIds'])) + for item in data + )) + + def __getitem__(self, key): + return self.children[key] + + def __iter__(self): + return iter(self.children.values()) + + +@attr.s +class LessonV1(object): + name = attr.ib() + id = attr.ib() + slug = attr.ib() + child_ids = attr.ib() + + def children(self, all_children): + return [all_children[child] for child in self.child_ids] + + +@attr.s +class LessonsV1(object): + children = attr.ib() + + @staticmethod + def from_json(data): + return LessonsV1(OrderedDict( + (item['id'], + LessonV1(item['name'], + item['id'], + item['slug'], + item['itemIds'])) + for item in data + )) + + def __getitem__(self, key): + return self.children[key] + + +@attr.s +class ItemV2(object): + name = attr.ib() + id = attr.ib() + slug = attr.ib() + type_name = attr.ib() + lesson_id = attr.ib() + module_id = attr.ib() + + +@attr.s +class ItemsV2(object): + children = attr.ib() + + @staticmethod + def from_json(data): + return ItemsV2({ + item['id']: + ItemV2(item['name'], + item['id'], + item['slug'], + item['contentSummary']['typeName'], + item['lessonId'], + item['moduleId']) + for item in data + }) + + def __getitem__(self, key): + return self.children[key] + + +@attr.s +class VideoV1(object): + resolution = attr.ib() + mp4_video_url = attr.ib() + + +@attr.s +class VideosV1(object): + children = attr.ib() + + @staticmethod + def from_json(data): + + videos = [VideoV1(resolution, links['mp4VideoUrl']) + for resolution, links + in data['sources']['byResolution'].items()] + videos.sort(key=lambda video: video.resolution, reverse=True) + + videos = OrderedDict( + (video.resolution, video) + for video in videos + ) + return VideosV1(videos) + + def __contains__(self, key): + return key in self.children + + def __getitem__(self, key): + return self.children[key] + + def get_best(self): + return next(iter(self.children.values())) + + class CourseraOnDemand(object): """ This is a class that provides a friendly interface to extract certain @@ -687,9 +820,9 @@ class CourseraOnDemand(object): }) return headers - def extract_links_from_lecture(self, + def extract_links_from_lecture(self, course_id, video_id, subtitle_language='en', - resolution='540p', assets=None): + resolution='540p'): """ Return the download URLs of on-demand course video. @@ -702,18 +835,13 @@ class CourseraOnDemand(object): @param resolution: Preferred video resolution. @type resolution: str - @param assets: List of assets that may present in the video. - @type assets: [str] - @return: @see CourseraOnDemand._extract_links_from_text """ - if assets is None: - assets = [] - try: links = self._extract_videos_and_subtitles_from_lecture( - video_id, subtitle_language, resolution) + course_id, video_id, subtitle_language, resolution) + assets = self._get_lecture_asset_ids(course_id, video_id) assets = self._normalize_assets(assets) extend_supplement_links( links, self._extract_links_from_lecture_assets(assets)) @@ -727,6 +855,17 @@ class CourseraOnDemand(object): 'Could not download lecture %s: %s', video_id, exception) return None + def _get_lecture_asset_ids(self, course_id, video_id): + """ + Obtain a list of asset ids from a lecture. + """ + dom = get_page(self._session, OPENCOURSE_ONDEMAND_LECTURE_ASSETS_URL, + json=True, course_id=course_id, video_id=video_id) + # Note that we extract here "id", not definition -> assetId, as it + # be extracted later. + return [asset['id'] + for asset in dom['linked']['openCourseAssets.v1']] + def _normalize_assets(self, assets): """ Perform asset normalization. For some reason, assets that are sometimes @@ -850,41 +989,34 @@ class CourseraOnDemand(object): return urls def _extract_videos_and_subtitles_from_lecture(self, + course_id, video_id, subtitle_language='en', resolution='540p'): - dom = get_page(self._session, OPENCOURSE_VIDEO_URL, - json=True, - video_id=video_id) - logging.debug('Parsing JSON for video_id <%s>.', video_id) + + dom = get_page(self._session, OPENCOURSE_ONDEMAND_LECTURE_VIDEOS_URL, + json=True, + course_id=course_id, + video_id=video_id) + dom = dom['linked']['onDemandVideos.v1'][0] + + videos = VideosV1.from_json(dom) video_content = {} - # videos - logging.debug('Gathering video URLs for video_id <%s>.', video_id) - sources = dom['sources'] - sources.sort(key=lambda src: src['resolution']) - sources.reverse() - - # Try to select resolution requested by the user. - filtered_sources = [source - for source in sources - if source['resolution'] == resolution] - - if len(filtered_sources) == 0: - # We will just use the 'vanilla' version of sources here, instead of - # filtered_sources. - logging.warning('Requested resolution %s not available for <%s>. ' - 'Downloading highest resolution available instead.', - resolution, video_id) - else: + if resolution in videos: + source = videos[resolution] logging.debug('Proceeding with download of resolution %s of <%s>.', resolution, video_id) - sources = filtered_sources + else: + source = videos.get_best() + logging.warning( + 'Requested resolution %s not available for <%s>. ' + 'Downloading highest resolution (%s) available instead.', + resolution, video_id, source.resolution) - video_url = sources[0]['formatSources']['video/mp4'] - video_content['mp4'] = video_url + video_content['mp4'] = source.mp4_video_url subtitle_link = self._extract_subtitles_from_video_dom( dom, subtitle_language, video_id) diff --git a/coursera/coursera_dl.py b/coursera/coursera_dl.py index 4c20689..a389106 100644 --- a/coursera/coursera_dl.py +++ b/coursera/coursera_dl.py @@ -67,7 +67,8 @@ from .workflow import CourseraDownloader from .parallel import ConsecutiveDownloader, ParallelDownloader from .utils import (clean_filename, get_anchor_format, mkdir_p, fix_url, print_ssl_error_message, - decode_input, BeautifulSoup, is_debug_run) + decode_input, BeautifulSoup, is_debug_run, + spit_json, slurp_json) from .network import get_page, get_page_and_url from .commandline import parse_args @@ -126,8 +127,7 @@ def download_on_demand_class(args, class_name): cached_syllabus_filename = '%s-syllabus-parsed.json' % class_name if args.cache_syllabus and os.path.isfile(cached_syllabus_filename): - with open(cached_syllabus_filename) as syllabus_file: - modules = json.load(syllabus_file) + modules = slurp_json(cached_syllabus_filename) else: error_occured, modules = extractor.get_modules( class_name, @@ -141,8 +141,7 @@ def download_on_demand_class(args, class_name): ) if is_debug_run or args.cache_syllabus(): - with open(cached_syllabus_filename, 'w') as file_object: - json.dump(modules, file_object, indent=4) + spit_json(modules, cached_syllabus_filename) if args.only_syllabus: return error_occured, False diff --git a/coursera/define.py b/coursera/define.py index e2ec105..ff8c51e 100644 --- a/coursera/define.py +++ b/coursera/define.py @@ -61,8 +61,10 @@ OPENCOURSE_LIST_COURSES = 'https://api.coursera.org/api/courses.v1?q=watchlist&s # } # } OPENCOURSE_MEMBERSHIPS = 'https://api.coursera.org/api/memberships.v1?includes=courseId,courses.v1&q=me&showHidden=true&filter=current,preEnrolled' -OPENCOURSE_CONTENT_URL = 'https://api.coursera.org/api/opencourse.v1/course/{class_name}?showLockedItems=true' -OPENCOURSE_VIDEO_URL = 'https://api.coursera.org/api/opencourse.v1/video/{video_id}' +OPENCOURSE_ONDEMAND_LECTURE_VIDEOS_URL = \ + 'https://api.coursera.org/api/onDemandLectureVideos.v1/'\ + '{course_id}~{video_id}?includes=video&'\ + 'fields=onDemandVideos.v1(sources%2Csubtitles%2CsubtitlesVtt%2CsubtitlesTxt)' OPENCOURSE_SUPPLEMENT_URL = 'https://api.coursera.org/api/onDemandSupplements.v1/'\ '{course_id}~{element_id}?includes=asset&fields=openCourseAssets.v1%28typeName%29,openCourseAssets.v1%28definition%29' OPENCOURSE_PROGRAMMING_ASSIGNMENTS_URL = \ @@ -97,6 +99,23 @@ OPENCOURSE_REFERENCE_ITEM_URL = \ OPENCOURSE_ASSET_URL = \ 'https://api.coursera.org/api/assetUrls.v1?ids={ids}' +# Sample response: +# "linked": { +# "openCourseAssets.v1": [ +# { +# "typeName": "asset", +# "definition": { +# "assetId": "fytYX5rYEeedWRLokafKRg", +# "name": "Lecture slides" +# }, +# "id": "j6g7VZrYEeeUVgpv-dYMig" +# } +# ] +# } +OPENCOURSE_ONDEMAND_LECTURE_ASSETS_URL = \ + 'https://api.coursera.org/api/onDemandLectureAssets.v1/'\ + '{course_id}~{video_id}/?includes=openCourseAssets' + # These ids are provided in lecture json: # # { @@ -170,9 +189,20 @@ OPENCOURSE_API_ASSETS_V1_URL = \ OPENCOURSE_ONDEMAND_COURSE_MATERIALS = \ 'https://api.coursera.org/api/onDemandCourseMaterials.v1/?'\ - 'q=slug&slug={class_name}&includes=moduleIds%2ClessonIds%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2CitemIds%2Ctracks'\ - '&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight)%2ConDemandCourseMaterialItems.v1(name%2Cslug%2CtimeCommitment%2Ccontent%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\ - '&showLockedItems=true' + 'q=slug&slug={class_name}&includes=moduleIds%2ClessonIds%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2CitemIds%2Ctracks'\ + '&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight)%2ConDemandCourseMaterialItems.v1(name%2Cslug%2CtimeCommitment%2Ccontent%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\ + '&showLockedItems=true' + +OPENCOURSE_ONDEMAND_COURSE_MATERIALS_V2 = \ + 'https://api.coursera.org/api/onDemandCourseMaterials.v2/?q=slug&slug={class_name}'\ + '&includes=modules%2Clessons%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2Citems%2Ctracks%2CgradePolicy&'\ + '&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional%2ClearningObjectives)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight%2CisRequiredForPassing)%2ConDemandCourseMaterialItems.v2(name%2Cslug%2CtimeCommitment%2CcontentSummary%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId%2ClockedStatus%2CitemLockSummary)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\ + '&showLockedItems=true' + +OPENCOURSE_ONDEMAND_COURSES_V1 = \ + 'https://api.coursera.org/api/onDemandCourses.v1?q=slug&slug={class_name}&'\ + 'includes=instructorIds%2CpartnerIds%2C_links&'\ + 'fields=brandingImage%2CcertificatePurchaseEnabledAt%2Cpartners.v1(squareLogo%2CrectangularLogo)%2Cinstructors.v1(fullName)%2CoverridePartnerLogos%2CsessionsEnabledAt%2CdomainTypes%2CpremiumExperienceVariant%2CisRestrictedMembership' ABOUT_URL = ('https://api.coursera.org/api/catalog.v1/courses?' 'fields=largeIcon,photo,previewLink,shortDescription,smallIcon,' @@ -924,7 +954,7 @@ pre {