Mirrors/coursera-dl
1
0
Fork 0
mirror of https://github.com/coursera-dl/coursera-dl.git synced 2026-07-17 16:35:35 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #677 from coursera-dl/new-api-for-syllabus-and-lectures

Browse source 
New api for syllabus and lectures
This commit is contained in:
Yuri Bochkarev 2018-06-24 10:59:47 +03:00 • committed by GitHub
commit dda61d2530
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 838 additions and 484 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.travis.yml
View file

@ -1,6 +1,5 @@
language: python
python:
- "2.6"
- "2.7"
- "3.4"
- "3.5"

7
CHANGELOG.md
View file

@ -1,5 +1,12 @@
# Change Log
Bugfixes:
- Switch to newer API for syllabus and lecture retrieval (#665, #673, #634)
Features:
- You can now download specializations: the child courses will be
downloaded automatically
## 0.11.2 (2018-06-03)
Bugfixes:

324
coursera/api.py
View file

@ -12,9 +12,11 @@ import logging
import time
import requests
import urllib
from collections import namedtuple
from collections import namedtuple, OrderedDict
from six import iterkeys, iteritems
from six.moves.urllib_parse import quote_plus
import attr
from .utils import (BeautifulSoup, make_coursera_absolute_url,
extend_supplement_links, clean_url, clean_filename,
@ -26,7 +28,11 @@ from .define import (OPENCOURSE_SUPPLEMENT_URL,
OPENCOURSE_ASSETS_URL,
OPENCOURSE_API_ASSETS_V1_URL,
OPENCOURSE_ONDEMAND_COURSE_MATERIALS,
OPENCOURSE_VIDEO_URL,
OPENCOURSE_ONDEMAND_COURSE_MATERIALS_V2,
OPENCOURSE_ONDEMAND_COURSES_V1,
OPENCOURSE_ONDEMAND_LECTURE_VIDEOS_URL,
OPENCOURSE_ONDEMAND_LECTURE_ASSETS_URL,
OPENCOURSE_ONDEMAND_SPECIALIZATIONS_V1,
OPENCOURSE_MEMBERSHIPS,
OPENCOURSE_REFERENCES_POLL_URL,
OPENCOURSE_REFERENCE_ITEM_URL,
@ -278,7 +284,7 @@ class MarkupToHTMLConverter(object):
audio.insert_after(controls_tag)
class OnDemandCourseMaterialItems(object):
class OnDemandCourseMaterialItemsV1(object):
"""
Helper class that allows accessing lecture JSONs by lesson IDs.
"""
@ -312,7 +318,7 @@ class OnDemandCourseMaterialItems(object):
dom = get_page(session, OPENCOURSE_ONDEMAND_COURSE_MATERIALS,
json=True,
class_name=course_name)
return OnDemandCourseMaterialItems(
return OnDemandCourseMaterialItemsV1(
dom['linked']['onDemandCourseMaterialItems.v1'])
def get(self, lesson_id):
@ -408,6 +414,173 @@ class AssetRetriever(object):
return result
@attr.s
class ModuleV1(object):
name = attr.ib()
id = attr.ib()
slug = attr.ib()
child_ids = attr.ib()
def children(self, all_children):
return [all_children[child] for child in self.child_ids]
@attr.s
class ModulesV1(object):
children = attr.ib()
@staticmethod
def from_json(data):
return ModulesV1(OrderedDict(
(item['id'],
ModuleV1(item['name'],
item['id'],
item['slug'],
item['lessonIds']))
for item in data
))
def __getitem__(self, key):
return self.children[key]
def __iter__(self):
return iter(self.children.values())
@attr.s
class LessonV1(object):
name = attr.ib()
id = attr.ib()
slug = attr.ib()
child_ids = attr.ib()
def children(self, all_children):
return [all_children[child] for child in self.child_ids]
@attr.s
class LessonsV1(object):
children = attr.ib()
@staticmethod
def from_json(data):
return LessonsV1(OrderedDict(
(item['id'],
LessonV1(item['name'],
item['id'],
item['slug'],
item['itemIds']))
for item in data
))
def __getitem__(self, key):
return self.children[key]
@attr.s
class ItemV2(object):
name = attr.ib()
id = attr.ib()
slug = attr.ib()
type_name = attr.ib()
lesson_id = attr.ib()
module_id = attr.ib()
@attr.s
class ItemsV2(object):
children = attr.ib()
@staticmethod
def from_json(data):
return ItemsV2(OrderedDict(
(item['id'],
ItemV2(item['name'],
item['id'],
item['slug'],
item['contentSummary']['typeName'],
item['lessonId'],
item['moduleId']))
for item in data
))
def __getitem__(self, key):
return self.children[key]
@attr.s
class VideoV1(object):
resolution = attr.ib()
mp4_video_url = attr.ib()
@attr.s
class VideosV1(object):
children = attr.ib()
@staticmethod
def from_json(data):
videos = [VideoV1(resolution, links['mp4VideoUrl'])
for resolution, links
in data['sources']['byResolution'].items()]
videos.sort(key=lambda video: video.resolution, reverse=True)
videos = OrderedDict(
(video.resolution, video)
for video in videos
)
return VideosV1(videos)
def __contains__(self, key):
return key in self.children
def __getitem__(self, key):
return self.children[key]
def get_best(self):
return next(iter(self.children.values()))
def expand_specializations(session, class_names):
"""
Checks whether any given name is not a class but a specialization.
If it's a specialization, expand the list of class names with the child
class names.
"""
result = []
for class_name in class_names:
specialization = SpecializationV1.create(session, class_name)
if specialization is None:
result.append(class_name)
else:
result.extend(specialization.children)
logging.info('Expanded specialization "%s" into the following'
' classes: %s',
class_name, ' '.join(specialization.children))
return result
@attr.s
class SpecializationV1(object):
children = attr.ib()
@staticmethod
def create(session, class_name):
try:
dom = get_page(session, OPENCOURSE_ONDEMAND_SPECIALIZATIONS_V1,
json=True, quiet=True,
class_name=class_name)
except requests.exceptions.HTTPError as e:
logging.debug('Could not expand %s: %s', class_name, e)
return None
return SpecializationV1(
[course['slug'] for course in dom['linked']['courses.v1']])
class CourseraOnDemand(object):
"""
This is a class that provides a friendly interface to extract certain
@ -481,87 +654,84 @@ class CourseraOnDemand(object):
supplement_links = {}
url = url.format(**kwargs)
reply = get_page(
self._session,
url,
json=True
)
headers = self._auth_headers_with_json()
reply = get_page(self._session, url, json=True)
for content in reply['content']:
if content['type'] == 'directory':
a = self._get_notebook_folder(
OPENCOURSE_NOTEBOOK_TREE, jupyterId, jupId=jupyterId, path=content['path'], timestamp=int(time.time()))
OPENCOURSE_NOTEBOOK_TREE, jupyterId, jupId=jupyterId,
path=content['path'], timestamp=int(time.time()))
supplement_links.update(a)
elif content['type'] == 'file':
tmpUrl = OPENCOURSE_NOTEBOOK_DOWNLOAD.format(
path=content['path'], jupId=jupyterId, timestamp=int(time.time()))
filename, extension = os.path.splitext(clean_url(tmpUrl))
tmp_url = OPENCOURSE_NOTEBOOK_DOWNLOAD.format(
path=content['path'], jupId=jupyterId,
timestamp=int(time.time()))
filename, extension = os.path.splitext(clean_url(tmp_url))
head, tail = os.path.split(content['path'])
# '/' in the following line is for a reason:
# @noureddin says: "I split head using split('/') not
# os.path.split() because it's seems to me that it comes from a
# web page, so the separator will always be /, so using the
# native path splitting function is not the most portable way to
# do it."
# Original pull request: https://github.com/coursera-dl/coursera-dl/pull/654
# native path splitting function is not the most portable
# way to do it."
# Original pull request:
# https://github.com/coursera-dl/coursera-dl/pull/654
head = '/'.join([clean_filename(dir, minimal_change=True)
for dir in head.split('/')])
tail = clean_filename(tail, minimal_change=True)
if os.path.isdir(self._course_name + "/notebook/" + head + "/") == False:
logging.info('Creating [{}] directories...'.format(head))
if not os.path.isdir(self._course_name + "/notebook/" + head + "/"):
logging.info('Creating [%s] directories...', head)
os.makedirs(self._course_name + "/notebook/" + head + "/")
r = requests.get(tmpUrl.replace(" ", "%20"),
r = requests.get(tmp_url.replace(" ", "%20"),
cookies=self._session.cookies)
if os.path.exists(self._course_name + "/notebook/" + head + "/" + tail) == False:
logging.info('Downloading {} into {}'.format(tail, head))
if not os.path.exists(self._course_name + "/notebook/" + head + "/" + tail):
logging.info('Downloading %s into %s', tail, head)
with open(self._course_name + "/notebook/" + head + "/" + tail, 'wb+') as f:
f.write(r.content)
else:
logging.info('Skipping {}... (file exists)'.format(tail))
logging.info('Skipping %s... (file exists)', tail)
if not str(extension[1:]) in supplement_links:
if str(extension[1:]) not in supplement_links:
supplement_links[str(extension[1:])] = []
supplement_links[str(extension[1:])].append(
(tmpUrl.replace(" ", "%20"), filename))
(tmp_url.replace(" ", "%20"), filename))
elif content['type'] == 'notebook':
tmpUrl = OPENCOURSE_NOTEBOOK_DOWNLOAD.format(
tmp_url = OPENCOURSE_NOTEBOOK_DOWNLOAD.format(
path=content['path'], jupId=jupyterId, timestamp=int(time.time()))
filename, extension = os.path.splitext(clean_url(tmpUrl))
filename, extension = os.path.splitext(clean_url(tmp_url))
head, tail = os.path.split(content['path'])
if os.path.isdir(self._course_name + "/notebook/" + head + "/") == False:
logging.info('Creating [{}] directories...'.format(head))
if not os.path.isdir(self._course_name + "/notebook/" + head + "/"):
logging.info('Creating [%s] directories...', head)
os.makedirs(self._course_name + "/notebook/" + head + "/")
r = requests.get(tmpUrl.replace(" ", "%20"),
r = requests.get(tmp_url.replace(" ", "%20"),
cookies=self._session.cookies)
if os.path.exists(self._course_name + "/notebook/" + head + "/" + tail) == False:
if not os.path.exists(self._course_name + "/notebook/" + head + "/" + tail):
logging.info(
'Downloading Jupyter {} into {}'.format(tail, head))
'Downloading Jupyter %s into %s', tail, head)
with open(self._course_name + "/notebook/" + head + "/" + tail, 'wb+') as f:
f.write(r.content)
else:
logging.info('Skipping {}... (file exists)'.format(tail))
logging.info('Skipping %s... (file exists)', tail)
if not "ipynb" in supplement_links:
if "ipynb" not in supplement_links:
supplement_links["ipynb"] = []
supplement_links["ipynb"].append(
(tmpUrl.replace(" ", "%20"), filename))
(tmp_url.replace(" ", "%20"), filename))
else:
logging.info(
'Unsupported typename {} in notebook'.format(content['type']))
'Unsupported typename %s in notebook', content['type'])
return supplement_links
@ -576,18 +746,21 @@ class CourseraOnDemand(object):
headers=headers
)
jupyterId = re.findall(r"\"\/user\/(.*)\/tree\"", reply)
if len(jupyterId) == 0:
jupyted_id = re.findall(r"\"\/user\/(.*)\/tree\"", reply)
if len(jupyted_id) == 0:
logging.error('Could not download notebook %s', notebook_id)
return None
jupyterId = jupyterId[0]
jupyted_id = jupyted_id[0]
newReq = requests.Session()
req = newReq.get(OPENCOURSE_NOTEBOOK_TREE.format(
jupId=jupyterId, path="/", timestamp=int(time.time())), headers=headers)
jupId=jupyted_id, path="/", timestamp=int(time.time())),
headers=headers)
return self._get_notebook_folder(OPENCOURSE_NOTEBOOK_TREE, jupyterId, jupId=jupyterId, path="/", timestamp=int(time.time()))
return self._get_notebook_folder(
OPENCOURSE_NOTEBOOK_TREE, jupyted_id, jupId=jupyted_id,
path="/", timestamp=int(time.time()))
def extract_links_from_notebook(self, notebook_id):
@ -687,9 +860,9 @@ class CourseraOnDemand(object):
})
return headers
def extract_links_from_lecture(self,
def extract_links_from_lecture(self, course_id,
video_id, subtitle_language='en',
resolution='540p', assets=None):
resolution='540p'):
"""
Return the download URLs of on-demand course video.
@ -702,18 +875,13 @@ class CourseraOnDemand(object):
@param resolution: Preferred video resolution.
@type resolution: str
@param assets: List of assets that may present in the video.
@type assets: [str]
@return: @see CourseraOnDemand._extract_links_from_text
"""
if assets is None:
assets = []
try:
links = self._extract_videos_and_subtitles_from_lecture(
video_id, subtitle_language, resolution)
course_id, video_id, subtitle_language, resolution)
assets = self._get_lecture_asset_ids(course_id, video_id)
assets = self._normalize_assets(assets)
extend_supplement_links(
links, self._extract_links_from_lecture_assets(assets))
@ -727,6 +895,17 @@ class CourseraOnDemand(object):
'Could not download lecture %s: %s', video_id, exception)
return None
def _get_lecture_asset_ids(self, course_id, video_id):
"""
Obtain a list of asset ids from a lecture.
"""
dom = get_page(self._session, OPENCOURSE_ONDEMAND_LECTURE_ASSETS_URL,
json=True, course_id=course_id, video_id=video_id)
# Note that we extract here "id", not definition -> assetId, as it
# be extracted later.
return [asset['id']
for asset in dom['linked']['openCourseAssets.v1']]
def _normalize_assets(self, assets):
"""
Perform asset normalization. For some reason, assets that are sometimes
@ -850,41 +1029,34 @@ class CourseraOnDemand(object):
return urls
def _extract_videos_and_subtitles_from_lecture(self,
course_id,
video_id,
subtitle_language='en',
resolution='540p'):
dom = get_page(self._session, OPENCOURSE_VIDEO_URL,
json=True,
video_id=video_id)
logging.debug('Parsing JSON for video_id <%s>.', video_id)
dom = get_page(self._session, OPENCOURSE_ONDEMAND_LECTURE_VIDEOS_URL,
json=True,
course_id=course_id,
video_id=video_id)
dom = dom['linked']['onDemandVideos.v1'][0]
videos = VideosV1.from_json(dom)
video_content = {}
# videos
logging.debug('Gathering video URLs for video_id <%s>.', video_id)
sources = dom['sources']
sources.sort(key=lambda src: src['resolution'])
sources.reverse()
# Try to select resolution requested by the user.
filtered_sources = [source
for source in sources
if source['resolution'] == resolution]
if len(filtered_sources) == 0:
# We will just use the 'vanilla' version of sources here, instead of
# filtered_sources.
logging.warning('Requested resolution %s not available for <%s>. '
'Downloading highest resolution available instead.',
resolution, video_id)
else:
if resolution in videos:
source = videos[resolution]
logging.debug('Proceeding with download of resolution %s of <%s>.',
resolution, video_id)
sources = filtered_sources
else:
source = videos.get_best()
logging.warning(
'Requested resolution %s not available for <%s>. '
'Downloading highest resolution (%s) available instead.',
resolution, video_id, source.resolution)
video_url = sources[0]['formatSources']['video/mp4']
video_content['mp4'] = video_url
video_content['mp4'] = source.mp4_video_url
subtitle_link = self._extract_subtitles_from_video_dom(
dom, subtitle_language, video_id)

603
coursera/commandline.py
View file

@ -47,338 +47,388 @@ def parse_args(args=None):
# Basic options
group_basic = parser.add_argument_group('Basic options')
group_basic.add_argument('class_names',
action='store',
nargs='*',
help='name(s) of the class(es) (e.g. "ml-005")')
group_basic.add_argument(
'class_names',
action='store',
nargs='*',
help='name(s) of the class(es) (e.g. "ml-005")')
group_basic.add_argument('-u',
'--username',
dest='username',
action='store',
default=None,
help='username (email) that you use to login to Coursera')
group_basic.add_argument(
'-u',
'--username',
dest='username',
action='store',
default=None,
help='username (email) that you use to login to Coursera')
group_basic.add_argument('-p',
'--password',
dest='password',
action='store',
default=None,
help='coursera password')
group_basic.add_argument(
'-p',
'--password',
dest='password',
action='store',
default=None,
help='coursera password')
group_basic.add_argument('--jobs',
dest='jobs',
action='store',
default=1,
type=int,
help='number of parallel jobs to use for '
'downloading resources. (Default: 1)')
group_basic.add_argument(
'--jobs',
dest='jobs',
action='store',
default=1,
type=int,
help='number of parallel jobs to use for '
'downloading resources. (Default: 1)')
group_basic.add_argument('--download-delay',
dest='download_delay',
action='store',
default=60,
type=int,
help='number of seconds to wait before downloading '
'next course. (Default: 60)')
group_basic.add_argument(
'--download-delay',
dest='download_delay',
action='store',
default=60,
type=int,
help='number of seconds to wait before downloading '
'next course. (Default: 60)')
group_basic.add_argument('-b', # FIXME: kill this one-letter option
'--preview',
dest='preview',
action='store_true',
default=False,
help='get videos from preview pages. (Default: False)')
group_basic.add_argument(
'-b', # FIXME: kill this one-letter option
'--preview',
dest='preview',
action='store_true',
default=False,
help='get videos from preview pages. (Default: False)')
group_basic.add_argument('--path',
dest='path',
action='store',
default='',
help='path to where to save the file. (Default: current directory)')
group_basic.add_argument(
'--path',
dest='path',
action='store',
default='',
help='path to where to save the file. (Default: current directory)')
group_basic.add_argument('-sl', # FIXME: deprecate this option
'--subtitle-language',
dest='subtitle_language',
action='store',
default='all',
help='Choose language to download subtitles and transcripts. (Default: all)'
'Use special value "all" to download all available.'
'To download subtitles and transcripts of multiple languages,'
'use comma(s) (without spaces) to separate the names of the languages, i.e., "en,zh-CN".'
'To download subtitles and transcripts of alternative language(s) '
'if only the current language is not available,'
'put an "|<lang>" for each of the alternative languages after '
'the current language, i.e., "en|fr,zh-CN|zh-TW|de", and make sure the parameter are wrapped with '
'quotes when "|" presents.'
)
group_basic.add_argument(
'-sl', # FIXME: deprecate this option
'--subtitle-language',
dest='subtitle_language',
action='store',
default='all',
help='Choose language to download subtitles and transcripts.'
'(Default: all) Use special value "all" to download all available.'
'To download subtitles and transcripts of multiple languages,'
'use comma(s) (without spaces) to seperate the names of the languages,'
' i.e., "en,zh-CN".'
'To download subtitles and transcripts of alternative language(s) '
'if only the current language is not available,'
'put an "|<lang>" for each of the alternative languages after '
'the current language, i.e., "en|fr,zh-CN|zh-TW|de", and make sure '
'the parameter are wrapped with quotes when "|" presents.'
)
# Selection of material to download
group_material = parser.add_argument_group(
'Selection of material to download')
group_material.add_argument('--only-syllabus',
dest='only_syllabus',
action='store_true',
default=False,
help='download only syllabus, skip course content. '
'(Default: False)')
group_material.add_argument(
'--only-syllabus',
dest='only_syllabus',
action='store_true',
default=False,
help='download only syllabus, skip course content. '
'(Default: False)')
group_material.add_argument('--download-quizzes',
dest='download_quizzes',
action='store_true',
default=False,
help='download quiz and exam questions. (Default: False)')
group_material.add_argument(
'--download-quizzes',
dest='download_quizzes',
action='store_true',
default=False,
help='download quiz and exam questions. (Default: False)')
group_material.add_argument('--download-notebooks',
dest='download_notebooks',
action='store_true',
default=False,
help='download Python Jupyter Notebooks. (Default: False)')
group_material.add_argument(
'--download-notebooks',
dest='download_notebooks',
action='store_true',
default=False,
help='download Python Jupyther Notebooks. (Default: False)')
group_material.add_argument('--about', # FIXME: should be --about-course
dest='about',
action='store_true',
default=False,
help='download "about" metadata. (Default: False)')
group_material.add_argument(
'--about', # FIXME: should be --about-course
dest='about',
action='store_true',
default=False,
help='download "about" metadata. (Default: False)')
group_material.add_argument('-f',
'--formats',
dest='file_formats',
action='store',
default='all',
help='file format extensions to be downloaded in'
' quotes space separated, e.g. "mp4 pdf" '
'(default: special value "all")')
group_material.add_argument(
'-f',
'--formats',
dest='file_formats',
action='store',
default='all',
help='file format extensions to be downloaded in'
' quotes space separated, e.g. "mp4 pdf" '
'(default: special value "all")')
group_material.add_argument('--ignore-formats',
dest='ignore_formats',
action='store',
default=None,
help='file format extensions of resources to ignore'
' (default: None)')
group_material.add_argument(
'--ignore-formats',
dest='ignore_formats',
action='store',
default=None,
help='file format extensions of resources to ignore'
' (default: None)')
group_material.add_argument('-sf', # FIXME: deprecate this option
'--section_filter',
dest='section_filter',
action='store',
default=None,
help='only download sections which contain this'
' regex (default: disabled)')
group_material.add_argument(
'-sf', # FIXME: deprecate this option
'--section_filter',
dest='section_filter',
action='store',
default=None,
help='only download sections which contain this'
' regex (default: disabled)')
group_material.add_argument('-lf', # FIXME: deprecate this option
'--lecture_filter',
dest='lecture_filter',
action='store',
default=None,
help='only download lectures which contain this regex'
' (default: disabled)')
group_material.add_argument(
'-lf', # FIXME: deprecate this option
'--lecture_filter',
dest='lecture_filter',
action='store',
default=None,
help='only download lectures which contain this regex'
' (default: disabled)')
group_material.add_argument('-rf', # FIXME: deprecate this option
'--resource_filter',
dest='resource_filter',
action='store',
default=None,
help='only download resources which match this regex'
' (default: disabled)')
group_material.add_argument(
'-rf', # FIXME: deprecate this option
'--resource_filter',
dest='resource_filter',
action='store',
default=None,
help='only download resources which match this regex'
' (default: disabled)')
group_material.add_argument('--video-resolution',
dest='video_resolution',
action='store',
default='540p',
help='video resolution to download (default: 540p); '
'only valid for on-demand courses; '
'only values allowed: 360p, 540p, 720p')
group_material.add_argument(
'--video-resolution',
dest='video_resolution',
action='store',
default='540p',
help='video resolution to download (default: 540p); '
'only valid for on-demand courses; '
'only values allowed: 360p, 540p, 720p')
group_material.add_argument('--disable-url-skipping',
dest='disable_url_skipping',
action='store_true',
default=False,
help='disable URL skipping, all URLs will be '
'downloaded (default: False)')
group_material.add_argument(
'--disable-url-skipping',
dest='disable_url_skipping',
action='store_true',
default=False,
help='disable URL skipping, all URLs will be '
'downloaded (default: False)')
# Parameters related to external downloaders
group_external_dl = parser.add_argument_group('External downloaders')
group_external_dl.add_argument('--wget',
dest='wget',
action='store',
nargs='?',
const='wget',
default=None,
help='use wget for downloading,'
'optionally specify wget bin')
group_external_dl.add_argument('--curl',
dest='curl',
action='store',
nargs='?',
const='curl',
default=None,
help='use curl for downloading,'
' optionally specify curl bin')
group_external_dl.add_argument('--aria2',
dest='aria2',
action='store',
nargs='?',
const='aria2c',
default=None,
help='use aria2 for downloading,'
' optionally specify aria2 bin')
group_external_dl.add_argument('--axel',
dest='axel',
action='store',
nargs='?',
const='axel',
default=None,
help='use axel for downloading,'
' optionally specify axel bin')
group_external_dl.add_argument('--downloader-arguments',
dest='downloader_arguments',
default='',
help='additional arguments passed to the'
' downloader')
group_external_dl.add_argument(
'--wget',
dest='wget',
action='store',
nargs='?',
const='wget',
default=None,
help='use wget for downloading,'
'optionally specify wget bin')
parser.add_argument('--list-courses',
dest='list_courses',
action='store_true',
default=False,
help='list course names (slugs) and quit. Listed '
'course names can be put into program arguments')
group_external_dl.add_argument(
'--curl',
dest='curl',
action='store',
nargs='?',
const='curl',
default=None,
help='use curl for downloading,'
' optionally specify curl bin')
parser.add_argument('--resume',
dest='resume',
action='store_true',
default=False,
help='resume incomplete downloads (default: False)')
group_external_dl.add_argument(
'--aria2',
dest='aria2',
action='store',
nargs='?',
const='aria2c',
default=None,
help='use aria2 for downloading,'
' optionally specify aria2 bin')
parser.add_argument('-o',
'--overwrite',
dest='overwrite',
action='store_true',
default=False,
help='whether existing files should be overwritten'
' (default: False)')
group_external_dl.add_argument(
'--axel',
dest='axel',
action='store',
nargs='?',
const='axel',
default=None,
help='use axel for downloading,'
' optionally specify axel bin')
parser.add_argument('--verbose-dirs',
dest='verbose_dirs',
action='store_true',
default=False,
help='include class name in section directory name')
group_external_dl.add_argument(
'--downloader-arguments',
dest='downloader_arguments',
default='',
help='additional arguments passed to the'
' downloader')
parser.add_argument('--quiet',
dest='quiet',
action='store_true',
default=False,
help='omit as many messages as possible'
' (only printing errors)')
parser.add_argument(
'--list-courses',
dest='list_courses',
action='store_true',
default=False,
help='list course names (slugs) and quit. Listed '
'course names can be put into program arguments')
parser.add_argument('-r',
'--reverse',
dest='reverse',
action='store_true',
default=False,
help='download sections in reverse order')
parser.add_argument(
'--resume',
dest='resume',
action='store_true',
default=False,
help='resume incomplete downloads (default: False)')
parser.add_argument('--combined-section-lectures-nums',
dest='combined_section_lectures_nums',
action='store_true',
default=False,
help='include lecture and section name in final files')
parser.add_argument(
'-o',
'--overwrite',
dest='overwrite',
action='store_true',
default=False,
help='whether existing files should be overwritten'
' (default: False)')
parser.add_argument('--unrestricted-filenames',
dest='unrestricted_filenames',
action='store_true',
default=False,
help='Do not limit filenames to be ASCII-only')
parser.add_argument(
'--verbose-dirs',
dest='verbose_dirs',
action='store_true',
default=False,
help='include class name in section directory name')
parser.add_argument(
'--quiet',
dest='quiet',
action='store_true',
default=False,
help='omit as many messages as possible'
' (only printing errors)')
parser.add_argument(
'-r',
'--reverse',
dest='reverse',
action='store_true',
default=False,
help='download sections in reverse order')
parser.add_argument(
'--combined-section-lectures-nums',
dest='combined_section_lectures_nums',
action='store_true',
default=False,
help='include lecture and section name in final files')
parser.add_argument(
'--unrestricted-filenames',
dest='unrestricted_filenames',
action='store_true',
default=False,
help='Do not limit filenames to be ASCII-only')
# Advanced authentication
group_adv_auth = parser.add_argument_group(
'Advanced authentication options')
group_adv_auth.add_argument('-c',
'--cookies_file',
dest='cookies_file',
action='store',
default=None,
help='full path to the cookies.txt file')
group_adv_auth.add_argument(
'-c',
'--cookies_file',
dest='cookies_file',
action='store',
default=None,
help='full path to the cookies.txt file')
group_adv_auth.add_argument('-n',
'--netrc',
dest='netrc',
nargs='?',
action='store',
const=True,
default=False,
help='use netrc for reading passwords, uses default'
' location if no path specified')
group_adv_auth.add_argument(
'-n',
'--netrc',
dest='netrc',
nargs='?',
action='store',
const=True,
default=False,
help='use netrc for reading passwords, uses default'
' location if no path specified')
group_adv_auth.add_argument('-k',
'--keyring',
dest='use_keyring',
action='store_true',
default=False,
help='use keyring provided by operating system to '
'save and load credentials')
group_adv_auth.add_argument(
'-k',
'--keyring',
dest='use_keyring',
action='store_true',
default=False,
help='use keyring provided by operating system to '
'save and load credentials')
group_adv_auth.add_argument('--clear-cache',
dest='clear_cache',
action='store_true',
default=False,
help='clear cached cookies')
group_adv_auth.add_argument(
'--clear-cache',
dest='clear_cache',
action='store_true',
default=False,
help='clear cached cookies')
# Advanced miscellaneous options
group_adv_misc = parser.add_argument_group(
'Advanced miscellaneous options')
group_adv_misc.add_argument('--hook',
dest='hooks',
action='append',
default=[],
help='hooks to run when finished')
group_adv_misc.add_argument(
'--hook',
dest='hooks',
action='append',
default=[],
help='hooks to run when finished')
group_adv_misc.add_argument('-pl',
'--playlist',
dest='playlist',
action='store_true',
default=False,
help='generate M3U playlists for course weeks')
group_adv_misc.add_argument(
'-pl',
'--playlist',
dest='playlist',
action='store_true',
default=False,
help='generate M3U playlists for course weeks')
group_adv_misc.add_argument('--mathjax-cdn',
dest='mathjax_cdn_url',
default='https://cdn.mathjax.org/mathjax/latest/MathJax.js',
help='the cdn address of MathJax.js'
)
group_adv_misc.add_argument(
'--mathjax-cdn',
dest='mathjax_cdn_url',
default='https://cdn.mathjax.org/mathjax/latest/MathJax.js',
help='the cdn address of MathJax.js'
)
# Debug options
group_debug = parser.add_argument_group('Debugging options')
group_debug.add_argument('--skip-download',
dest='skip_download',
action='store_true',
default=False,
help='for debugging: skip actual downloading of files')
group_debug.add_argument(
'--skip-download',
dest='skip_download',
action='store_true',
default=False,
help='for debugging: skip actual downloading of files')
group_debug.add_argument('--debug',
dest='debug',
action='store_true',
default=False,
help='print lots of debug information')
group_debug.add_argument(
'--debug',
dest='debug',
action='store_true',
default=False,
help='print lots of debug information')
group_debug.add_argument('--cache-syllabus',
dest='cache_syllabus',
action='store_true',
default=False,
help='cache course syllabus into a file')
group_debug.add_argument(
'--cache-syllabus',
dest='cache_syllabus',
action='store_true',
default=False,
help='cache course syllabus into a file')
group_debug.add_argument('--version',
dest='version',
action='store_true',
default=False,
help='display version and exit')
group_debug.add_argument(
'--version',
dest='version',
action='store_true',
default=False,
help='display version and exit')
group_debug.add_argument('-l', # FIXME: remove short option from rarely used ones
'--process_local_page',
dest='local_page',
help='uses or creates local cached version of syllabus'
' page')
group_debug.add_argument(
'-l', # FIXME: remove short option from rarely used ones
'--process_local_page',
dest='local_page',
help='uses or creates local cached version of syllabus'
' page')
# Final parsing of the options
args = parser.parse_args(args)
@ -403,7 +453,8 @@ def parse_args(args=None):
# show version?
if args.version:
# we use print (not logging) function because version may be used
# by some external script while logging may output excessive information
# by some external script while logging may output excessive
# information
print(__version__)
sys.exit(0)

42
coursera/coursera_dl.py
View file

@ -60,15 +60,17 @@ import requests
from .cookies import (
AuthenticationFailed, ClassNotFound,
get_cookies_for_class, make_cookie_values, TLSAdapter)
get_cookies_for_class, make_cookie_values, TLSAdapter, login)
from .define import (CLASS_URL, ABOUT_URL, PATH_CACHE)
from .downloaders import get_downloader
from .workflow import CourseraDownloader
from .parallel import ConsecutiveDownloader, ParallelDownloader
from .utils import (clean_filename, get_anchor_format, mkdir_p, fix_url,
print_ssl_error_message,
decode_input, BeautifulSoup, is_debug_run)
decode_input, BeautifulSoup, is_debug_run,
spit_json, slurp_json)
from .api import expand_specializations
from .network import get_page, get_page_and_url
from .commandline import parse_args
from .extractors import CourseraExtractor
@ -103,16 +105,18 @@ def list_courses(args):
@type args: namedtuple
"""
session = get_session()
extractor = CourseraExtractor(session, args.username, args.password)
login(session, args.username, args.password)
extractor = CourseraExtractor(session)
courses = extractor.list_courses()
logging.info('Found %d courses', len(courses))
for course in courses:
logging.info(course)
def download_on_demand_class(args, class_name):
def download_on_demand_class(session, args, class_name):
"""
Download all requested resources from the on-demand class given in class_name.
Download all requested resources from the on-demand class given
in class_name.
@return: Tuple of (bool, bool), where the first bool indicates whether
errors occurred while parsing syllabus, the second bool indicates
@ -121,13 +125,11 @@ def download_on_demand_class(args, class_name):
"""
error_occurred = False
session = get_session()
extractor = CourseraExtractor(session, args.username, args.password)
extractor = CourseraExtractor(session)
cached_syllabus_filename = '%s-syllabus-parsed.json' % class_name
if args.cache_syllabus and os.path.isfile(cached_syllabus_filename):
with open(cached_syllabus_filename) as syllabus_file:
modules = json.load(syllabus_file)
modules = slurp_json(cached_syllabus_filename)
else:
error_occurred, modules = extractor.get_modules(
class_name,
@ -141,8 +143,7 @@ def download_on_demand_class(args, class_name):
)
if is_debug_run or args.cache_syllabus():
with open(cached_syllabus_filename, 'w') as file_object:
json.dump(modules, file_object, indent=4)
spit_json(modules, cached_syllabus_filename)
if args.only_syllabus:
return error_occurred, False
@ -200,7 +201,7 @@ def print_failed_urls(failed_urls):
logging.info('-' * 80)
def download_class(args, class_name):
def download_class(session, args, class_name):
"""
Try to download on-demand class.
@ -210,7 +211,7 @@ def download_class(args, class_name):
@rtype: (bool, bool)
"""
logging.debug('Downloading new style (on demand) class %s', class_name)
return download_on_demand_class(args, class_name)
return download_on_demand_class(session, args, class_name)
def main():
@ -231,11 +232,16 @@ def main():
list_courses(args)
return
session = get_session()
login(session, args.username, args.password)
args.class_names = expand_specializations(session, args.class_names)
for class_index, class_name in enumerate(args.class_names):
try:
logging.info('Downloading class: %s (%d / %d)',
class_name, class_index + 1, len(args.class_names))
error_occurred, completed = download_class(args, class_name)
error_occurred, completed = download_class(
session, args, class_name)
if completed:
completed_classes.append(class_name)
if error_occurred:
@ -249,10 +255,10 @@ def main():
print_ssl_error_message(e)
if is_debug_run():
raise
except ClassNotFound as cnf:
logging.error('Could not find class: %s', cnf)
except AuthenticationFailed as af:
logging.error('Could not authenticate: %s', af)
except ClassNotFound as e:
logging.error('Could not find class: %s', e)
except AuthenticationFailed as e:
logging.error('Could not authenticate: %s', e)
if class_index + 1 != len(args.class_names):
logging.info('Sleeping for %d seconds before downloading next course. '

57
coursera/define.py
View file

@ -61,8 +61,10 @@ OPENCOURSE_LIST_COURSES = 'https://api.coursera.org/api/courses.v1?q=watchlist&s
# }
# }
OPENCOURSE_MEMBERSHIPS = 'https://api.coursera.org/api/memberships.v1?includes=courseId,courses.v1&q=me&showHidden=true&filter=current,preEnrolled'
OPENCOURSE_CONTENT_URL = 'https://api.coursera.org/api/opencourse.v1/course/{class_name}?showLockedItems=true'
OPENCOURSE_VIDEO_URL = 'https://api.coursera.org/api/opencourse.v1/video/{video_id}'
OPENCOURSE_ONDEMAND_LECTURE_VIDEOS_URL = \
'https://api.coursera.org/api/onDemandLectureVideos.v1/'\
'{course_id}~{video_id}?includes=video&'\
'fields=onDemandVideos.v1(sources%2Csubtitles%2CsubtitlesVtt%2CsubtitlesTxt)'
OPENCOURSE_SUPPLEMENT_URL = 'https://api.coursera.org/api/onDemandSupplements.v1/'\
'{course_id}~{element_id}?includes=asset&fields=openCourseAssets.v1%28typeName%29,openCourseAssets.v1%28definition%29'
OPENCOURSE_PROGRAMMING_ASSIGNMENTS_URL = \
@ -97,6 +99,23 @@ OPENCOURSE_REFERENCE_ITEM_URL = \
OPENCOURSE_ASSET_URL = \
'https://api.coursera.org/api/assetUrls.v1?ids={ids}'
# Sample response:
# "linked": {
# "openCourseAssets.v1": [
# {
# "typeName": "asset",
# "definition": {
# "assetId": "fytYX5rYEeedWRLokafKRg",
# "name": "Lecture slides"
# },
# "id": "j6g7VZrYEeeUVgpv-dYMig"
# }
# ]
# }
OPENCOURSE_ONDEMAND_LECTURE_ASSETS_URL = \
'https://api.coursera.org/api/onDemandLectureAssets.v1/'\
'{course_id}~{video_id}/?includes=openCourseAssets'
# These ids are provided in lecture json:
#
# {
@ -170,9 +189,35 @@ OPENCOURSE_API_ASSETS_V1_URL = \
OPENCOURSE_ONDEMAND_COURSE_MATERIALS = \
'https://api.coursera.org/api/onDemandCourseMaterials.v1/?'\
'q=slug&slug={class_name}&includes=moduleIds%2ClessonIds%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2CitemIds%2Ctracks'\
'&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight)%2ConDemandCourseMaterialItems.v1(name%2Cslug%2CtimeCommitment%2Ccontent%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\
'&showLockedItems=true'
'q=slug&slug={class_name}&includes=moduleIds%2ClessonIds%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2CitemIds%2Ctracks'\
'&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight)%2ConDemandCourseMaterialItems.v1(name%2Cslug%2CtimeCommitment%2Ccontent%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\
'&showLockedItems=true'
OPENCOURSE_ONDEMAND_COURSE_MATERIALS_V2 = \
'https://api.coursera.org/api/onDemandCourseMaterials.v2/?q=slug&slug={class_name}'\
'&includes=modules%2Clessons%2CpassableItemGroups%2CpassableItemGroupChoices%2CpassableLessonElements%2Citems%2Ctracks%2CgradePolicy&'\
'&fields=moduleIds%2ConDemandCourseMaterialModules.v1(name%2Cslug%2Cdescription%2CtimeCommitment%2ClessonIds%2Coptional%2ClearningObjectives)%2ConDemandCourseMaterialLessons.v1(name%2Cslug%2CtimeCommitment%2CelementIds%2Coptional%2CtrackId)%2ConDemandCourseMaterialPassableItemGroups.v1(requiredPassedCount%2CpassableItemGroupChoiceIds%2CtrackId)%2ConDemandCourseMaterialPassableItemGroupChoices.v1(name%2Cdescription%2CitemIds)%2ConDemandCourseMaterialPassableLessonElements.v1(gradingWeight%2CisRequiredForPassing)%2ConDemandCourseMaterialItems.v2(name%2Cslug%2CtimeCommitment%2CcontentSummary%2CisLocked%2ClockableByItem%2CitemLockedReasonCode%2CtrackId%2ClockedStatus%2CitemLockSummary)%2ConDemandCourseMaterialTracks.v1(passablesCount)'\
'&showLockedItems=true'
OPENCOURSE_ONDEMAND_SPECIALIZATIONS_V1 = \
'https://api.coursera.org/api/onDemandSpecializations.v1?q=slug'\
'&slug={class_name}&fields=courseIds,interchangeableCourseIds,launchedAt,'\
'logo,memberships,metadata,partnerIds,premiumExperienceVariant,'\
'onDemandSpecializationMemberships.v1(suggestedSessionSchedule),'\
'onDemandSpecializationSuggestedSchedule.v1(suggestedSessions),'\
'partners.v1(homeLink,name),courses.v1(courseProgress,description,'\
'membershipIds,startDate,v2Details,vcMembershipIds),v2Details.v1('\
'onDemandSessions,plannedLaunchDate),memberships.v1(grade,'\
'vcMembershipId),vcMemberships.v1(certificateCodeWithGrade)'\
'&includes=courseIds,memberships,partnerIds,'\
'onDemandSpecializationMemberships.v1(suggestedSessionSchedule),'\
'courses.v1(courseProgress,membershipIds,v2Details,vcMembershipIds),'\
'v2Details.v1(onDemandSessions)'
OPENCOURSE_ONDEMAND_COURSES_V1 = \
'https://api.coursera.org/api/onDemandCourses.v1?q=slug&slug={class_name}&'\
'includes=instructorIds%2CpartnerIds%2C_links&'\
'fields=brandingImage%2CcertificatePurchaseEnabledAt%2Cpartners.v1(squareLogo%2CrectangularLogo)%2Cinstructors.v1(fullName)%2CoverridePartnerLogos%2CsessionsEnabledAt%2CdomainTypes%2CpremiumExperienceVariant%2CisRestrictedMembership'
ABOUT_URL = ('https://api.coursera.org/api/catalog.v1/courses?'
'fields=largeIcon,photo,previewLink,shortDescription,smallIcon,'
@ -924,7 +969,7 @@ pre {
<script type="text/javascript" async
src="'''
INSTRUCTIONS_HTML_MATHJAX_URL = 'https://cdn.mathjax.org/mathjax/latest/MathJax.js'
INSTRUCTIONS_HTML_INJECTION_AFTER ='''?config=TeX-AMS-MML_HTMLorMML">
INSTRUCTIONS_HTML_INJECTION_AFTER = '''?config=TeX-AMS-MML_HTMLorMML">
</script>
<script type="text/x-mathjax-config">

140
coursera/extractors.py
View file

@ -9,11 +9,11 @@ import abc
import json
import logging
from .api import CourseraOnDemand, OnDemandCourseMaterialItems
from .define import OPENCOURSE_CONTENT_URL
from .cookies import login
from .api import (CourseraOnDemand, OnDemandCourseMaterialItemsV1,
ModulesV1, LessonsV1, ItemsV2)
from .define import OPENCOURSE_ONDEMAND_COURSE_MATERIALS_V2
from .network import get_page
from .utils import is_debug_run
from .utils import is_debug_run, spit_json
class PlatformExtractor(object):
@ -27,8 +27,7 @@ class PlatformExtractor(object):
class CourseraExtractor(PlatformExtractor):
def __init__(self, session, username, password):
login(session, username, password)
def __init__(self, session):
self._notebook_downloaded = False
self._session = session
@ -52,9 +51,11 @@ class CourseraExtractor(PlatformExtractor):
page = self._get_on_demand_syllabus(class_name)
error_occurred, modules = self._parse_on_demand_syllabus(
class_name,
page, reverse, unrestricted_filenames,
subtitle_language, video_resolution,
download_quizzes, mathjax_cdn_url, download_notebooks)
return error_occurred, modules
def _get_on_demand_syllabus(self, class_name):
@ -62,13 +63,14 @@ class CourseraExtractor(PlatformExtractor):
Get the on-demand course listing webpage.
"""
url = OPENCOURSE_CONTENT_URL.format(class_name=class_name)
url = OPENCOURSE_ONDEMAND_COURSE_MATERIALS_V2.format(
class_name=class_name)
page = get_page(self._session, url)
logging.info('Downloaded %s (%d bytes)', url, len(page))
logging.debug('Downloaded %s (%d bytes)', url, len(page))
return page
def _parse_on_demand_syllabus(self, page, reverse=False,
def _parse_on_demand_syllabus(self, course_name, page, reverse=False,
unrestricted_filenames=False,
subtitle_language='en',
video_resolution=None,
@ -86,115 +88,132 @@ class CourseraExtractor(PlatformExtractor):
"""
dom = json.loads(page)
course_name = dom['slug']
class_id = dom['elements'][0]['id']
logging.info('Parsing syllabus of on-demand course. '
'This may take some time, please be patient ...')
logging.info('Parsing syllabus of on-demand course (id=%s). '
'This may take some time, please be patient ...',
class_id)
modules = []
json_modules = dom['courseMaterial']['elements']
course = CourseraOnDemand(session=self._session, course_id=dom['id'],
course_name=course_name,
unrestricted_filenames=unrestricted_filenames,
mathjax_cdn_url=mathjax_cdn_url
)
json_modules = dom['linked']['onDemandCourseMaterialItems.v2']
course = CourseraOnDemand(
session=self._session, course_id=class_id,
course_name=course_name,
unrestricted_filenames=unrestricted_filenames,
mathjax_cdn_url=mathjax_cdn_url)
course.obtain_user_id()
ondemand_material_items = OnDemandCourseMaterialItems.create(
ondemand_material_items = OnDemandCourseMaterialItemsV1.create(
session=self._session, course_name=course_name)
if is_debug_run():
with open('%s-syllabus-raw.json' % course_name, 'w') as file_object:
json.dump(dom, file_object, indent=4)
with open('%s-course-material-items.json' % course_name, 'w') as file_object:
json.dump(ondemand_material_items._items, file_object, indent=4)
spit_json(dom, '%s-syllabus-raw.json' % course_name)
spit_json(json_modules, '%s-material-items-v2.json' % course_name)
spit_json(ondemand_material_items._items,
'%s-course-material-items.json' % course_name)
error_occurred = False
for module in json_modules:
module_slug = module['slug']
logging.info('Processing module %s', module_slug)
sections = []
json_sections = module['elements']
for section in json_sections:
section_slug = section['slug']
logging.info('Processing section %s', section_slug)
all_modules = ModulesV1.from_json(
dom['linked']['onDemandCourseMaterialModules.v1'])
all_lessons = LessonsV1.from_json(
dom['linked']['onDemandCourseMaterialLessons.v1'])
all_items = ItemsV2.from_json(
dom['linked']['onDemandCourseMaterialItems.v2'])
for module in all_modules:
logging.info('Processing module %s', module.slug)
lessons = []
for section in module.children(all_lessons):
logging.info('Processing section %s', section.slug)
lectures = []
json_lectures = section['elements']
available_lectures = section.children(all_items)
# Certain modules may be empty-looking programming assignments
# e.g. in data-structures, algorithms-on-graphs ondemand courses
if not json_lectures:
lesson_id = section['id']
lecture = ondemand_material_items.get(lesson_id)
# e.g. in data-structures, algorithms-on-graphs ondemand
# courses
if not available_lectures:
lecture = ondemand_material_items.get(section.id)
if lecture is not None:
json_lectures = [lecture]
available_lectures = [lecture]
for lecture in json_lectures:
lecture_slug = lecture['slug']
typename = lecture['content']['typeName']
for lecture in available_lectures:
typename = lecture.type_name
logging.info('Processing lecture %s (%s)',
lecture_slug, typename)
lecture.slug, typename)
# Empty dictionary means there were no data
# None means an error occurred
links = {}
if typename == 'lecture':
lecture_video_id = lecture['content']['definition']['videoId']
assets = lecture['content']['definition'].get('assets', [])
# lecture_video_id = lecture['content']['definition']['videoId']
# assets = lecture['content']['definition'].get(
# 'assets', [])
lecture_video_id = lecture.id
# assets = []
links = course.extract_links_from_lecture(
class_id,
lecture_video_id, subtitle_language,
video_resolution, assets)
video_resolution)
elif typename == 'supplement':
links = course.extract_links_from_supplement(lecture['id'])
links = course.extract_links_from_supplement(
lecture.id)
elif typename == 'phasedPeer':
links = course.extract_links_from_peer_assignment(lecture['id'])
links = course.extract_links_from_peer_assignment(
lecture.id)
elif typename in ('gradedProgramming', 'ungradedProgramming'):
links = course.extract_links_from_programming(lecture['id'])
links = course.extract_links_from_programming(
lecture.id)
elif typename == 'quiz':
if download_quizzes:
links = course.extract_links_from_quiz(lecture['id'])
links = course.extract_links_from_quiz(
lecture.id)
elif typename == 'exam':
if download_quizzes:
links = course.extract_links_from_exam(lecture['id'])
links = course.extract_links_from_exam(
lecture.id)
elif typename == 'programming':
if download_quizzes:
links = course.extract_links_from_programming_immediate_instructions(lecture['id'])
links = course.extract_links_from_programming_immediate_instructions(
lecture.id)
elif typename == 'notebook':
if download_notebooks and self._notebook_downloaded == False:
logging.warning('According to notebooks platform, content will be downloaded first')
links = course.extract_links_from_notebook(lecture['id'])
if download_notebooks and not self._notebook_downloaded:
logging.warning(
'According to notebooks platform, content will be downloaded first')
links = course.extract_links_from_notebook(
lecture.id)
self._notebook_downloaded = True
else:
logging.info(
'Unsupported typename "%s" in lecture "%s" (lecture id "%s")',
typename, lecture_slug, lecture['id'])
typename, lecture.slug, lecture.id)
continue
if links is None:
error_occurred = True
elif links:
lectures.append((lecture_slug, links))
lectures.append((lecture.slug, links))
if lectures:
sections.append((section_slug, lectures))
lessons.append((section.slug, lectures))
if sections:
modules.append((module_slug, sections))
if lessons:
modules.append((module.slug, lessons))
if modules and reverse:
modules.reverse()
# Processing resources section
json_references= course.extract_references_poll()
json_references = course.extract_references_poll()
references = []
if json_references:
logging.info('Processing resources')
@ -204,7 +223,8 @@ class CourseraExtractor(PlatformExtractor):
logging.info('Processing resource %s',
reference_slug)
links = course.extract_links_from_reference(json_reference['shortId'])
links = course.extract_links_from_reference(
json_reference['shortId'])
if links is None:
error_occurred = True
elif links:

15
coursera/network.py
View file

@ -9,7 +9,7 @@ import logging
import requests
def get_reply(session, url, post=False, data=None, headers=None):
def get_reply(session, url, post=False, data=None, headers=None, quiet=False):
"""
Download an HTML page using the requests session. Low-level function
that allows for flexible request configuration.
@ -29,6 +29,10 @@ def get_reply(session, url, post=False, data=None, headers=None):
@param headers: Additional headers to send with request.
@type headers: dict
@param quiet: Flag that tells whether to print error message when status
code != 200.
@type quiet: bool
@return: Requests response.
@rtype: requests.Response
"""
@ -46,8 +50,9 @@ def get_reply(session, url, post=False, data=None, headers=None):
try:
reply.raise_for_status()
except requests.exceptions.HTTPError as e:
logging.error("Error %s getting page %s", e, url)
logging.error("The server replied: %s", reply.text)
if not quiet:
logging.error("Error %s getting page %s", e, url)
logging.error("The server replied: %s", reply.text)
raise
return reply
@ -59,6 +64,7 @@ def get_page(session,
post=False,
data=None,
headers=None,
quiet=False,
**kwargs):
"""
Download an HTML page using the requests session.
@ -82,7 +88,8 @@ def get_page(session,
@rtype: str
"""
url = url.format(**kwargs)
reply = get_reply(session, url, post=post, data=data, headers=headers)
reply = get_reply(session, url, post=post, data=data, headers=headers,
quiet=quiet)
return reply.json() if json else reply.text

117
coursera/test/test_api.py vendored
View file

@ -73,7 +73,7 @@ def test_extract_links_from_lecture_http_error(get_page, course):
locked_response.status_code = define.HTTP_FORBIDDEN
get_page.side_effect = HTTPError('Mocked HTTP error',
response=locked_response)
assert None == course.extract_links_from_lecture('0')
assert None == course.extract_links_from_lecture('fake_course_id', '0')
@patch('coursera.api.get_page')
@ -132,7 +132,8 @@ def test_extract_links_from_programming_immediate_instructions_http_error(
@patch('coursera.api.get_page')
def test_ondemand_programming_supplement_no_instructions(get_page, course):
no_instructions = slurp_fixture('json/supplement-programming-no-instructions.json')
no_instructions = slurp_fixture(
'json/supplement-programming-no-instructions.json')
get_page.return_value = json.loads(no_instructions)
output = course.extract_links_from_programming('0')
@ -171,7 +172,8 @@ def test_ondemand_from_programming_immediate_instructions_no_instructions(
@patch('coursera.api.get_page')
def test_ondemand_programming_supplement_empty_instructions(get_page, course):
empty_instructions = slurp_fixture('json/supplement-programming-empty-instructions.json')
empty_instructions = slurp_fixture(
'json/supplement-programming-empty-instructions.json')
get_page.return_value = json.loads(empty_instructions)
output = course.extract_links_from_programming('0')
@ -186,7 +188,7 @@ def test_ondemand_programming_supplement_empty_instructions(get_page, course):
@patch('coursera.api.get_page')
def test_ondemand_programming_immediate_instructions_empty_instructions(
get_page, course):
get_page, course):
empty_instructions = slurp_fixture(
'json/supplement-programming-immediate-instructions-empty-instructions.json')
get_page.return_value = json.loads(empty_instructions)
@ -207,10 +209,10 @@ def test_ondemand_programming_supplement_one_asset(get_page, course):
one_asset_url = slurp_fixture('json/asset-urls-one.json')
asset_json = json.loads(one_asset_url)
get_page.side_effect = [json.loads(one_asset_tag),
json.loads(one_asset_url)]
json.loads(one_asset_url)]
expected_output = {'pdf': [(asset_json['elements'][0]['url'],
'statement-pca')]}
'statement-pca')]}
output = course.extract_links_from_programming('0')
# Make sure that SOME html content has been extracted, but remove
@ -238,14 +240,15 @@ def test_extract_references_poll(get_page, course):
@patch('coursera.api.get_page')
def test_ondemand_programming_immediate_instructions_one_asset(get_page, course):
one_asset_tag = slurp_fixture('json/supplement-programming-immediate-instructions-one-asset.json')
one_asset_tag = slurp_fixture(
'json/supplement-programming-immediate-instructions-one-asset.json')
one_asset_url = slurp_fixture('json/asset-urls-one.json')
asset_json = json.loads(one_asset_url)
get_page.side_effect = [json.loads(one_asset_tag),
json.loads(one_asset_url)]
json.loads(one_asset_url)]
expected_output = {'pdf': [(asset_json['elements'][0]['url'],
'statement-pca')]}
'statement-pca')]}
output = course.extract_links_from_programming_immediate_instructions('0')
# Make sure that SOME html content has been extracted, but remove
@ -259,12 +262,14 @@ def test_ondemand_programming_immediate_instructions_one_asset(get_page, course)
@patch('coursera.api.get_page')
def test_ondemand_programming_supplement_three_assets(get_page, course):
three_assets_tag = slurp_fixture('json/supplement-programming-three-assets.json')
three_assets_tag = slurp_fixture(
'json/supplement-programming-three-assets.json')
three_assets_url = slurp_fixture('json/asset-urls-three.json')
get_page.side_effect = [json.loads(three_assets_tag),
json.loads(three_assets_url)]
json.loads(three_assets_url)]
expected_output = json.loads(slurp_fixture('json/supplement-three-assets-output.json'))
expected_output = json.loads(slurp_fixture(
'json/supplement-three-assets-output.json'))
output = course.extract_links_from_programming('0')
output = json.loads(json.dumps(output))
@ -279,12 +284,15 @@ def test_ondemand_programming_supplement_three_assets(get_page, course):
@patch('coursera.api.get_page')
def test_extract_links_from_lecture_assets_typename_asset(get_page, course):
open_course_assets_reply = slurp_fixture('json/supplement-open-course-assets-reply.json')
api_assets_v1_reply = slurp_fixture('json/supplement-api-assets-v1-reply.json')
open_course_assets_reply = slurp_fixture(
'json/supplement-open-course-assets-reply.json')
api_assets_v1_reply = slurp_fixture(
'json/supplement-api-assets-v1-reply.json')
get_page.side_effect = [json.loads(open_course_assets_reply),
json.loads(api_assets_v1_reply)]
json.loads(api_assets_v1_reply)]
expected_output = json.loads(slurp_fixture('json/supplement-extract-links-from-lectures-output.json'))
expected_output = json.loads(slurp_fixture(
'json/supplement-extract-links-from-lectures-output.json'))
assets = ['giAxucdaEeWJTQ5WTi8YJQ']
output = course._extract_links_from_lecture_assets(assets)
output = json.loads(json.dumps(output))
@ -298,14 +306,20 @@ def test_extract_links_from_lecture_assets_typname_url_and_asset(get_page, cours
links both from typename == 'asset' and == 'url'.
"""
get_page.side_effect = [
json.loads(slurp_fixture('json/supplement-open-course-assets-typename-url-reply-1.json')),
json.loads(slurp_fixture('json/supplement-open-course-assets-typename-url-reply-2.json')),
json.loads(slurp_fixture('json/supplement-open-course-assets-typename-url-reply-3.json')),
json.loads(slurp_fixture('json/supplement-open-course-assets-typename-url-reply-4.json')),
json.loads(slurp_fixture('json/supplement-open-course-assets-typename-url-reply-5.json')),
json.loads(slurp_fixture(
'json/supplement-open-course-assets-typename-url-reply-1.json')),
json.loads(slurp_fixture(
'json/supplement-open-course-assets-typename-url-reply-2.json')),
json.loads(slurp_fixture(
'json/supplement-open-course-assets-typename-url-reply-3.json')),
json.loads(slurp_fixture(
'json/supplement-open-course-assets-typename-url-reply-4.json')),
json.loads(slurp_fixture(
'json/supplement-open-course-assets-typename-url-reply-5.json')),
]
expected_output = json.loads(slurp_fixture('json/supplement-extract-links-from-lectures-url-asset-output.json'))
expected_output = json.loads(slurp_fixture(
'json/supplement-extract-links-from-lectures-url-asset-output.json'))
assets = ['Yry0spSKEeW8oA5fR3afVQ',
'kMQyUZSLEeWj-hLVp2Pm8w',
'xkAloZmJEeWjYA4jOOgP8Q']
@ -322,7 +336,8 @@ def test_list_courses(get_page, course):
get_page.side_effect = [
json.loads(slurp_fixture('json/list-courses-input.json'))
]
expected_output = json.loads(slurp_fixture('json/list-courses-output.json'))
expected_output = json.loads(
slurp_fixture('json/list-courses-output.json'))
expected_output = expected_output['courses']
output = course.list_courses()
assert expected_output == output
@ -344,12 +359,13 @@ def test_list_courses(get_page, course):
'en,zh-CN|zh-TW', "None"),
]
)
def test_extract_subtitles_from_video_dom(input_filename,output_filename,subtitle_language, video_id):
def test_extract_subtitles_from_video_dom(input_filename, output_filename, subtitle_language, video_id):
video_dom = json.loads(slurp_fixture('json/%s' % input_filename))
expected_output = json.loads(slurp_fixture('json/%s' % output_filename))
course = api.CourseraOnDemand(
session=Mock(cookies={}), course_id='0', course_name='test_course')
actual_output = course._extract_subtitles_from_video_dom(video_dom, subtitle_language, video_id)
actual_output = course._extract_subtitles_from_video_dom(
video_dom, subtitle_language, video_id)
actual_output = json.loads(json.dumps(actual_output))
assert actual_output == expected_output
@ -357,22 +373,29 @@ def test_extract_subtitles_from_video_dom(input_filename,output_filename,subtitl
@pytest.mark.parametrize(
"input_filename,output_filename", [
('empty-input.json', 'empty-output.txt'),
('answer-text-replaced-with-span-input.json', 'answer-text-replaced-with-span-output.txt'),
('question-type-textExactMatch-input.json', 'question-type-textExactMatch-output.txt'),
('answer-text-replaced-with-span-input.json',
'answer-text-replaced-with-span-output.txt'),
('question-type-textExactMatch-input.json',
'question-type-textExactMatch-output.txt'),
('question-type-regex-input.json', 'question-type-regex-output.txt'),
('question-type-mathExpression-input.json', 'question-type-mathExpression-output.txt'),
('question-type-mathExpression-input.json',
'question-type-mathExpression-output.txt'),
('question-type-checkbox-input.json', 'question-type-checkbox-output.txt'),
('question-type-mcq-input.json', 'question-type-mcq-output.txt'),
('question-type-singleNumeric-input.json', 'question-type-singleNumeric-output.txt'),
('question-type-singleNumeric-input.json',
'question-type-singleNumeric-output.txt'),
('question-type-reflect-input.json', 'question-type-reflect-output.txt'),
('question-type-mcqReflect-input.json', 'question-type-mcqReflect-output.txt'),
('question-type-mcqReflect-input.json',
'question-type-mcqReflect-output.txt'),
('question-type-unknown-input.json', 'question-type-unknown-output.txt'),
('multiple-questions-input.json', 'multiple-questions-output.txt'),
]
)
def test_quiz_exam_to_markup_converter(input_filename, output_filename):
quiz_json = json.loads(slurp_fixture('json/quiz-to-markup/%s' % input_filename))
expected_output = slurp_fixture('json/quiz-to-markup/%s' % output_filename).strip()
quiz_json = json.loads(slurp_fixture(
'json/quiz-to-markup/%s' % input_filename))
expected_output = slurp_fixture(
'json/quiz-to-markup/%s' % output_filename).strip()
converter = api.QuizExamToMarkupConverter(session=None)
actual_output = converter(quiz_json).strip()
@ -411,7 +434,8 @@ class TestMarkupToHTMLConverter:
<meta charset="UTF-8"/>
"""
assert self._p(markup) + self.STYLE == output
assert self._p(markup) + self.STYLE_WITH_ALTER == output_with_alter_mjcdn
assert self._p(markup) + \
self.STYLE_WITH_ALTER == output_with_alter_mjcdn
def test_replace_text_tag(self):
markup = """
@ -438,7 +462,8 @@ class TestMarkupToHTMLConverter:
output = self.markup_to_html(markup)
output_with_alter_mjcdn = self.markup_to_html_with_alter_mjcdn(markup)
assert self._p(result) + self.STYLE == output
assert self._p(result) + self.STYLE_WITH_ALTER == output_with_alter_mjcdn
assert self._p(result) + \
self.STYLE_WITH_ALTER == output_with_alter_mjcdn
def test_replace_heading(self):
output = self.markup_to_html("""
@ -501,7 +526,8 @@ class TestMarkupToHTMLConverter:
'nodata': Mock(data=None, content_type='image/png')
}
mock_asset_retriever.__call__ = Mock(return_value=None)
mock_asset_retriever.__getitem__ = Mock(side_effect=replies.__getitem__)
mock_asset_retriever.__getitem__ = Mock(
side_effect=replies.__getitem__)
self.markup_to_html._asset_retriever = mock_asset_retriever
output = self.markup_to_html("""
@ -532,7 +558,8 @@ class TestMarkupToHTMLConverter:
'bWTK9sYwEeW7AxLLCrgDQQ': Mock(data=b'b', content_type='unknown')
}
mock_asset_retriever.__call__ = Mock(return_value=None)
mock_asset_retriever.__getitem__ = Mock(side_effect=replies.__getitem__)
mock_asset_retriever.__getitem__ = Mock(
side_effect=replies.__getitem__)
self.markup_to_html._asset_retriever = mock_asset_retriever
output = self.markup_to_html("""
@ -570,6 +597,7 @@ def test_quiz_converter():
with open('quiz.html', 'w') as file:
file.write(result)
def test_quiz_converter_all():
pytest.skip()
import os
@ -583,8 +611,8 @@ def test_quiz_converter_all():
markup_to_html = api.MarkupToHTMLConverter(session=session)
path = 'quiz_json'
for filename in ['quiz-audio.json']: #os.listdir(path):
# for filename in ['all_question_types.json']:
for filename in ['quiz-audio.json']: # os.listdir(path):
# for filename in ['all_question_types.json']:
# if 'YV0W4' not in filename:
# continue
# if 'QVHj1' not in filename:
@ -600,6 +628,7 @@ def test_quiz_converter_all():
with open('quiz_html/' + filename + '.html', 'w') as f:
f.write(result)
def create_session():
from coursera.coursera_dl import get_session
from coursera.credentials import get_credentials
@ -625,10 +654,14 @@ def test_asset_retriever(get_reply, get_page):
'vdqUTz61Eea_CQ5dfWSAjQ']
expected_output = [
api.Asset(id="bWTK9sYwEeW7AxLLCrgDQQ", name="M111.mp3", type_name="audio", url="url4", content_type="image/png", data="<...>"),
api.Asset(id="VceKeChKEeaOMw70NkE3iw", name="09_graph_decomposition_problems_1.pdf", type_name="pdf", url="url7", content_type="image/png", data="<...>"),
api.Asset(id="VcmGXShKEea4ehL5RXz3EQ", name="09_graph_decomposition_starter_files_1.zip", type_name="generic", url="url2", content_type="image/png", data="<...>"),
api.Asset(id="vdqUTz61Eea_CQ5dfWSAjQ", name="Capture.PNG", type_name="image", url="url9", content_type="image/png", data="<...>"),
api.Asset(id="bWTK9sYwEeW7AxLLCrgDQQ", name="M111.mp3", type_name="audio",
url="url4", content_type="image/png", data="<...>"),
api.Asset(id="VceKeChKEeaOMw70NkE3iw", name="09_graph_decomposition_problems_1.pdf",
type_name="pdf", url="url7", content_type="image/png", data="<...>"),
api.Asset(id="VcmGXShKEea4ehL5RXz3EQ", name="09_graph_decomposition_starter_files_1.zip",
type_name="generic", url="url2", content_type="image/png", data="<...>"),
api.Asset(id="vdqUTz61Eea_CQ5dfWSAjQ", name="Capture.PNG",
type_name="image", url="url9", content_type="image/png", data="<...>"),
]
retriever = api.AssetRetriever(session=None)

15
coursera/utils.py
View file

@ -8,6 +8,7 @@ import os
import re
import sys
import time
import json
import errno
import random
import string
@ -41,7 +42,9 @@ else:
from .define import COURSERA_URL, WINDOWS_UNC_PREFIX
# Force us of bs4 with html.parser
BeautifulSoup = lambda page: BeautifulSoup_(page, 'html.parser')
def BeautifulSoup(page): return BeautifulSoup_(page, 'html.parser')
if six.PY2:
@ -55,6 +58,16 @@ else:
return x
def spit_json(obj, filename):
with open(filename, 'w') as file_object:
json.dump(obj, file_object, indent=4)
def slurp_json(filename):
with open(filename) as file_object:
return json.load(file_object)
def is_debug_run():
"""
Check whether we're running with DEBUG loglevel.

1
requirements.txt
View file

@ -5,3 +5,4 @@ urllib3>=1.10
pyasn1>=0.1.7
keyring>=4.0
configargparse>=0.12.0
attrs==18.1.0