Mirrors/coursera-dl
1
0
Fork 0
mirror of https://github.com/coursera-dl/coursera-dl.git synced 2026-01-23 02:35:37 +00:00
Code Issues Projects Releases Wiki Activity

resolve merge conflicts

Browse source
This commit is contained in:
Yuri Bochkarev 2018-06-24 00:28:25 +03:00
commit 95666a6fdf
9 changed files with 41 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

8
README.md
View file

@ -120,7 +120,7 @@ particular courses that you want to use with `coursera-dl`.
## Recommended installation method for all Operating Systems
From a command line (preferrably, from a virtual environment), simply issue
From a command line (preferably, from a virtual environment), simply issue
the command:
pip install coursera-dl
@ -327,7 +327,7 @@ where the script is supposed to be executed, with the following format:
#--mathjax-cdn https://cdn.bootcss.com/mathjax/2.7.1/MathJax.js
# more other parameters
Parameter which is stored in the file will be overriden if it is again specifed
Parameter which is stored in the file will be overriden if it is again specified
in your commandline script
**Note:** In `coursera-dl.conf`, all the parameters should not be wrapped
@ -337,11 +337,11 @@ with quotes.
In default mode when you interrupt the download process by pressing
<kbd>CTRL</kbd>+<kbd>C</kbd>, partially downloaded files will be deleted from your disk and
you have to start the download process from the begining. If your
you have to start the download process from the beginning. If your
download was interrupted by something other than KeyboardInterrupt
(<kbd>CTRL</kbd>+<kbd>C</kbd>) like sudden system crash, partially downloaded files will
remain on your disk and the next time you start the process again,
these files will be discraded from download list!, therefore it's your
these files will be discarded from download list!, therefore it's your
job to delete them manually before next start. For this reason we
added an option called `--resume` which continues your downloads from
where they stopped:

6
coursera/api.py
View file

@ -57,7 +57,7 @@ from .define import (OPENCOURSE_SUPPLEMENT_URL,
IN_MEMORY_MARKER)
from .cookies import prepape_auth_headers
from .cookies import prepare_auth_headers
class QuizExamToMarkupConverter(object):
@ -814,7 +814,7 @@ class CourseraOnDemand(object):
return reply['contentResponseBody']['session']['id']
def _auth_headers_with_json(self):
headers = prepape_auth_headers(self._session, include_cauth=True)
headers = prepare_auth_headers(self._session, include_cauth=True)
headers.update({
'Content-Type': 'application/json; charset=UTF-8'
})
@ -1075,7 +1075,7 @@ class CourseraOnDemand(object):
", ".join(subtitle_set_nonexist), video_id,
subtitle_description)
if not subtitle_set_download:
logging.warning("%s all requested subtitles are unavaliable,"
logging.warning("%s all requested subtitles are unavailable,"
"with video id: [%s], falling back to 'en' "
"%s", subtitle_description.capitalize(),
video_id,

14
coursera/cookies.py
View file

@ -25,7 +25,7 @@ from .utils import mkdir_p, random_string
# Monkey patch cookielib.Cookie.__init__.
# Reason: The expires value may be a decimal string,
# but the Cookie class uses int() ...
__orginal_init__ = cookielib.Cookie.__init__
__original_init__ = cookielib.Cookie.__init__
def __fixed_init__(self, version, name, value,
@ -41,7 +41,7 @@ def __fixed_init__(self, version, name, value,
rfc2109=False):
if expires is not None:
expires = float(expires)
__orginal_init__(self, version, name, value,
__original_init__(self, version, name, value,
port, port_specified,
domain, domain_specified, domain_initial_dot,
path, path_specified,
@ -69,15 +69,15 @@ class AuthenticationFailed(BaseException):
"""
def prepape_auth_headers(session, include_cauth=False):
def prepare_auth_headers(session, include_cauth=False):
"""
This function prepapes headers with CSRF/CAUTH tokens that can
This function prepares headers with CSRF/CAUTH tokens that can
be used in POST requests such as login/get_quiz.
@param session: Requests session.
@type session: requests.Session
@param include_cauth: Flag that indicates whethe CAUTH cookies should be
@param include_cauth: Flag that indicates whether CAUTH cookies should be
included as well.
@type include_cauth: bool
@ -133,7 +133,7 @@ def login(session, username, password, class_name=None):
logging.error(e)
raise ClassNotFound(class_name)
headers = prepape_auth_headers(session, include_cauth=False)
headers = prepare_auth_headers(session, include_cauth=False)
data = {
'email': username,
@ -355,7 +355,7 @@ def get_cookies_for_class(session, class_name,
Get the cookies for the given class.
We do not validate the cookies if they are loaded from a cookies file
because this is intented for debugging purposes or if the coursera
because this is intended for debugging purposes or if the coursera
authentication process has changed.
"""
if cookies_file:

16
coursera/coursera_dl.py
View file

@ -116,12 +116,12 @@ def download_on_demand_class(args, class_name):
Download all requested resources from the on-demand class given in class_name.
@return: Tuple of (bool, bool), where the first bool indicates whether
errors occured while parsing syllabus, the second bool indicaters
errors occurred while parsing syllabus, the second bool indicates
whether the course appears to be completed.
@rtype: (bool, bool)
"""
error_occured = False
error_occurred = False
session = get_session()
extractor = CourseraExtractor(session, args.username, args.password)
@ -129,7 +129,7 @@ def download_on_demand_class(args, class_name):
if args.cache_syllabus and os.path.isfile(cached_syllabus_filename):
modules = slurp_json(cached_syllabus_filename)
else:
error_occured, modules = extractor.get_modules(
error_occurred, modules = extractor.get_modules(
class_name,
args.reverse,
args.unrestricted_filenames,
@ -144,7 +144,7 @@ def download_on_demand_class(args, class_name):
spit_json(modules, cached_syllabus_filename)
if args.only_syllabus:
return error_occured, False
return error_occurred, False
downloader = get_downloader(session, class_name, args)
downloader_wrapper = ParallelDownloader(downloader, args.jobs) \
@ -176,7 +176,7 @@ def download_on_demand_class(args, class_name):
if course_downloader.failed_urls:
print_failed_urls(course_downloader.failed_urls)
return error_occured, completed
return error_occurred, completed
def print_skipped_urls(skipped_urls):
@ -204,7 +204,7 @@ def download_class(args, class_name):
Try to download on-demand class.
@return: Tuple of (bool, bool), where the first bool indicates whether
errors occured while parsing syllabus, the second bool indicaters
errors occurred while parsing syllabus, the second bool indicates
whether the course appears to be completed.
@rtype: (bool, bool)
"""
@ -234,10 +234,10 @@ def main():
try:
logging.info('Downloading class: %s (%d / %d)',
class_name, class_index + 1, len(args.class_names))
error_occured, completed = download_class(args, class_name)
error_occurred, completed = download_class(args, class_name)
if completed:
completed_classes.append(class_name)
if error_occured:
if error_occurred:
classes_with_errors.append(class_name)
except requests.exceptions.HTTPError as e:
logging.error('HTTPError %s', e)

3
coursera/credentials.py
View file

@ -134,7 +134,8 @@ def authenticate_through_netrc(path=None):
error_messages = '\n'.join(str(e) for e in errors)
raise CredentialsError(
'Did not find valid netrc file:\n' + error_messages)
'Did not find valid netrc file:\n' + error_messages +
'\nPlease run this command: chmod og-rw ~/.netrc')
def get_credentials(username=None, password=None, netrc=None, use_keyring=False):

4
coursera/define.py
View file

@ -16,7 +16,7 @@ AUTH_URL = 'https://accounts.coursera.org/api/v1/login'
AUTH_URL_V3 = 'https://api.coursera.org/api/login/v3'
CLASS_URL = 'https://class.coursera.org/{class_name}'
# The following link is left just for illustative purposes:
# The following link is left just for illustrative purposes:
# https://api.coursera.org/api/courses.v1?fields=display%2CpartnerIds%2CphotoUrl%2CstartDate%2Cpartners.v1(homeLink%2Cname)&includes=partnerIds&q=watchlist&start=0
# Reply is as follows:
# {
@ -36,7 +36,7 @@ CLASS_URL = 'https://class.coursera.org/{class_name}'
# }
OPENCOURSE_LIST_COURSES = 'https://api.coursera.org/api/courses.v1?q=watchlist&start={start}'
# The following link is left just for illustative purposes:
# The following link is left just for illustrative purposes:
# https://api.coursera.org/api/memberships.v1?fields=courseId,enrolledTimestamp,grade,id,lastAccessedTimestamp,onDemandSessionMembershipIds,onDemandSessionMemberships,role,v1SessionId,vc,vcMembershipId,courses.v1(courseStatus,display,partnerIds,photoUrl,specializations,startDate,v1Details,v2Details),partners.v1(homeLink,name),v1Details.v1(sessionIds),v1Sessions.v1(active,certificatesReleased,dbEndDate,durationString,hasSigTrack,startDay,startMonth,startYear),v2Details.v1(onDemandSessions,plannedLaunchDate,sessionsEnabledAt),specializations.v1(logo,name,partnerIds,shortName)&includes=courseId,onDemandSessionMemberships,vcMembershipId,courses.v1(partnerIds,specializations,v1Details,v2Details),v1Details.v1(sessionIds),v2Details.v1(onDemandSessions),specializations.v1(partnerIds)&q=me&showHidden=true&filter=current,preEnrolled
# Sample reply:
# {

14
coursera/extractors.py
View file

@ -52,13 +52,13 @@ class CourseraExtractor(PlatformExtractor):
download_notebooks=False):
page = self._get_on_demand_syllabus(class_name)
error_occured, modules = self._parse_on_demand_syllabus(
error_occurred, modules = self._parse_on_demand_syllabus(
class_name,
page, reverse, unrestricted_filenames,
subtitle_language, video_resolution,
download_quizzes, mathjax_cdn_url, download_notebooks)
return error_occured, modules
return error_occurred, modules
def _get_on_demand_syllabus(self, class_name):
"""
@ -113,7 +113,7 @@ class CourseraExtractor(PlatformExtractor):
spit_json(ondemand_material_items._items,
'%s-course-material-items.json' % course_name)
error_occured = False
error_occurred = False
all_modules = ModulesV1.from_json(
dom['linked']['onDemandCourseMaterialModules.v1'])
@ -144,7 +144,7 @@ class CourseraExtractor(PlatformExtractor):
logging.info('Processing lecture %s (%s)',
lecture.slug, typename)
# Empty dictionary means there were no data
# None means an error occured
# None means an error occurred
links = {}
if typename == 'lecture':
@ -201,7 +201,7 @@ class CourseraExtractor(PlatformExtractor):
continue
if links is None:
error_occured = True
error_occurred = True
elif links:
lectures.append((lecture.slug, links))
@ -228,7 +228,7 @@ class CourseraExtractor(PlatformExtractor):
links = course.extract_links_from_reference(
json_reference['shortId'])
if links is None:
error_occured = True
error_occurred = True
elif links:
reference.append(('', links))
@ -238,4 +238,4 @@ class CourseraExtractor(PlatformExtractor):
if references:
modules.append(("Resources", references))
return error_occured, modules
return error_occurred, modules

12
coursera/test/test_api.py vendored
View file

@ -150,7 +150,7 @@ def test_ondemand_programming_supplement_no_instructions(get_page, course):
('peer-assignment-no-instructions.json', ''),
]
)
def test_ondemand_from_peer_assgnment_instructions(
def test_ondemand_from_peer_assignment_instructions(
get_page, course, input_filename, expected_output):
instructions = slurp_fixture('json/%s' % input_filename)
get_page.return_value = json.loads(instructions)
@ -178,7 +178,7 @@ def test_ondemand_programming_supplement_empty_instructions(get_page, course):
output = course.extract_links_from_programming('0')
# Make sure that SOME html content has been extracted, but remove
# it immeditely because it's a hassle to properly prepare test input
# it immediately because it's a hassle to properly prepare test input
# for it. FIXME later.
assert 'html' in output
del output['html']
@ -195,7 +195,7 @@ def test_ondemand_programming_immediate_instructions_empty_instructions(
output = course.extract_links_from_programming_immediate_instructions('0')
# Make sure that SOME html content has been extracted, but remove
# it immeditely because it's a hassle to properly prepare test input
# it immediately because it's a hassle to properly prepare test input
# for it. FIXME later.
assert 'html' in output
del output['html']
@ -216,7 +216,7 @@ def test_ondemand_programming_supplement_one_asset(get_page, course):
output = course.extract_links_from_programming('0')
# Make sure that SOME html content has been extracted, but remove
# it immeditely because it's a hassle to properly prepare test input
# it immediately because it's a hassle to properly prepare test input
# for it. FIXME later.
assert 'html' in output
del output['html']
@ -252,7 +252,7 @@ def test_ondemand_programming_immediate_instructions_one_asset(get_page, course)
output = course.extract_links_from_programming_immediate_instructions('0')
# Make sure that SOME html content has been extracted, but remove
# it immeditely because it's a hassle to properly prepare test input
# it immediately because it's a hassle to properly prepare test input
# for it. FIXME later.
assert 'html' in output
del output['html']
@ -274,7 +274,7 @@ def test_ondemand_programming_supplement_three_assets(get_page, course):
output = json.loads(json.dumps(output))
# Make sure that SOME html content has been extracted, but remove
# it immeditely because it's a hassle to properly prepare test input
# it immediately because it's a hassle to properly prepare test input
# for it. FIXME later.
assert 'html' in output
del output['html']

4
coursera/test/test_parsing.py vendored
View file

@ -65,7 +65,7 @@ def test_that_we_parse_and_write_json_correctly(get_page, json_path):
def get_old_style_video(monkeypatch):
pytest.skip()
"""
Mock some methods that would, otherwise, create repeateadly many web
Mock some methods that would, otherwise, create repeatedly many web
requests.
More specifically, we mock:
@ -139,7 +139,7 @@ def test_get_on_demand_supplement_url_accumulates_assets(mocked):
output = course.extract_links_from_supplement('element_id')
# Make sure that SOME html content has been extracted, but remove
# it immeditely because it's a hassle to properly prepare test input
# it immediately because it's a hassle to properly prepare test input
# for it. FIXME later.
assert 'html' in output
del output['html']