diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..c6e43d8 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,7 @@ +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 4 +insert_final_newline = true diff --git a/.travis.yml b/.travis.yml index 7e132f0..53ee630 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,10 @@ language: python python: - "2.7" - - "3.4" - - "3.5" - "3.6" + - "3.7" + - "3.8" + - "3.9" - "pypy" matrix: allow_failures: diff --git a/CHANGELOG.md b/CHANGELOG.md index eea9a00..1aae0cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log +## 0.11.5 (2019-12-16) + +Features: + - add --cauth argument to specify CAUTH cookie directly from command-line (#724) + ## 0.11.4 (2018-06-24) Features: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0eca116..2895dd5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ developers/maintainers feel good when trying to change code that other people contributed. For the record, when this document mentions "I", it mostly means Rogério -Brito's (@rbrito) is the one to blame. +Theodoro de Brito's (@rbrito) is the one to blame. # Write good commit messages @@ -237,10 +237,8 @@ DRAFT `git add ... & git ci -m 'Bump version (old_version -> new_version)'` 4. `git tag new_version` 5. `git push && git push --tags` -6. `pandoc --from=markdown --to=rst --output=README.rst README.md`. - I think this is required for PyPI description to look nice. -7. `python setup.py sdist bdist_wheel --universal` to build the package -8. `twine upload dist/coursera-dl-0.6.1.tar.gz` to deploy the package. +6. `python setup.py sdist bdist_wheel --universal` to build the package +7. `twine upload dist/coursera-dl-0.6.1.tar.gz` to deploy the package. ## Docker diff --git a/README.md b/README.md index a0059f8..49601c5 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ - [Docker](#docker) - [Windows](#windows) - [Create an account with Coursera](#create-an-account-with-coursera) - - [Running the script](#running-the-script) +- [Running the script](#running-the-script) - [Resuming downloads](#resuming-downloads) - [Troubleshooting](#troubleshooting) - [China issues](#china-issues) @@ -101,11 +101,11 @@ relevant excerpt: # Installation instructions `coursera-dl` requires Python 2 or Python 3 and a free Coursera account -enrolled in the class of interest. (As of February of 2016, we test -automatically the execution of the program with Python versions 2.6, 2.7, -Pypy, 3.4, 3.5, and 3.6). +enrolled in the class of interest. (As of February of 2020, we test +automatically the execution of the program with Python versions 2.7, Pypy, +3.6, 3.7, 3.8, and 3.9). -**Note:** We *strongly* recommend that you use a Python 3 interpreter (3.4 +**Note:** We *strongly* recommend that you use a Python 3 interpreter (3.9 or later). On any operating system, ensure that the Python executable location is added @@ -145,7 +145,7 @@ installed in your system (or they can interfere with `coursera-dl`). Prefer to use the option `--user` to `pip install`, if you need can. **Note 2:** As already mentioned, we *strongly* recommend that you use a new -Python 3 interpreter (e.g., 3.4 or later), since Python 3 has better support +Python 3 interpreter (e.g., 3.9 or later), since Python 3 has better support for SSL/TLS (for secure connections) than earlier versions.
If you must use Python 2, be sure that you have at least Python 2.7.9 (later versions are OK).
@@ -254,7 +254,7 @@ Settings > Environment Variables. ``` Example: -C:\Python35\Scripts\;C:\Python35\; +C:\Python39\Scripts\;C:\Python39\; ``` Or if you have restricted installation permissions and you've installed Python @@ -262,7 +262,7 @@ under AppData, add this to your PATH. ``` Example: -C:\Users\\AppData\Local\Programs\Python\Python35-32\Scripts;C:\Users\\AppData\Local\Programs\Python\Python35-32; +C:\Users\\AppData\Local\Programs\Python\Python39-32\Scripts;C:\Users\\AppData\Local\Programs\Python\Python39-32; ``` Coursera-dl can now be run from commandline or powershell. @@ -272,16 +272,25 @@ Coursera-dl can now be run from commandline or powershell. If you don't already have one, create a [Coursera][1] account and enroll in a class. See https://www.coursera.org/courses for the list of classes. -## Running the script +# Running the script + +Refer to `coursera-dl --help` for a complete, up-to-date reference on the runtime options +supported by this utility. Run the script to download the materials by providing your Coursera account credentials (e.g. email address and password or a `~/.netrc` file), the class names, as well as any additional parameters: - +``` General: coursera-dl -u -p modelthinking-004 -If you don't want to type your password in command line as plain text, you can use the script without `-p` option. In this case you will be prompted for password once the script is run. + With CAUTH parameter: coursera-dl -ca 'some-ca-value-from-browser' modelthinking-004 +``` +If you don't want to type your password in command line as plain text, you can use the +script without `-p` option. In this case you will be prompted for password once the +script is run. +Here are some examples of how to invoke `coursera-dl` from the command line: +``` Without -p field: coursera-dl -u modelthinking-004 Multiple classes: coursera-dl -u -p saas historyofrock1-001 algo-2012-002 Filter by section name: coursera-dl -u -p -sf "Chapter_Four" crypto-004 @@ -289,27 +298,35 @@ If you don't want to type your password in command line as plain text, you can u Download only ppt files: coursera-dl -u -p -f "ppt" qcomp-2012-001 Use a ~/.netrc file: coursera-dl -n -- matrix-001 Get the preview classes: coursera-dl -n -b ni-001 + Download videos at 720p: coursera-dl -n --video-resolution 720p ni-001 Specify download path: coursera-dl -n --path=C:\Coursera\Classes\ comnetworks-002 Display help: coursera-dl --help Maintain a list of classes in a dir: Initialize: mkdir -p CURRENT/{class1,class2,..classN} Update: coursera-dl -n --path CURRENT `\ls CURRENT` - +``` **Note:** If your `ls` command is aliased to display a colorized output, you may experience problems. Be sure to escape the `ls` command (use `\ls`) to assure that no special characters get sent to the script. -Note that we *do* support the New Platform ("on-demand") classes. +Note that we *do* support the New Platform ("on-demand") courses. + +By default, videos are downloaded at 540p resolution. For on-demand courses, the +`--video-resolution` flag accepts 360p, 540p, and 720p values. + +To download just the `.txt` and/or `.srt` subtitle files instead of the videos, +use `-ignore-formats mp4 --subtitle-language en` or whatever format the videos +are encoded in and desired languages for subtitles. On \*nix platforms, the use of a `~/.netrc` file is a good alternative to specifying both your username (i.e., your email address) and password every time on the command line. To use it, simply add a line like the one below to a file named `.netrc` in your home directory (or the [equivalent][8], if you are using Windows) with contents like: - +``` machine coursera-dl login password - +``` Create the file if it doesn't exist yet. From then on, you can switch from using `-u` and `-p` to simply call `coursera-dl` with the option `-n` instead. This is especially convenient, as typing usernames (email @@ -319,16 +336,16 @@ more if you happened to choose a "strong" password). Alternatively, if you want to store your preferred parameters (which might also include your username and password), create a file named `coursera-dl.conf` where the script is supposed to be executed, with the following format: - +``` --username --password --subtitle-language en,zh-CN|zh-TW --download-quizzes #--mathjax-cdn https://cdn.bootcss.com/mathjax/2.7.1/MathJax.js # more other parameters - -Parameter which is stored in the file will be overriden if it is again specified -in your commandline script +``` +Parameters which are specified in the file will be overriden if they are +provided again on the commandline. **Note:** In `coursera-dl.conf`, all the parameters should not be wrapped with quotes. @@ -457,7 +474,8 @@ file that lists all the course materials. Maybe your friend who is enrolled could save that course page for you. In that case use the `--process_local_page` option. -Alternatively you may want to try this Chrome extension: https://chrome.google.com/webstore/detail/coursera-materials-downlo/ijkboagofaehocnjacacdhdcbbcpilih +Alternatively you may want to try this various browser extensions designed for +this problem. If none of the above works for you, there is nothing we can do. @@ -499,7 +517,7 @@ Related discussion: [#205](https://github.com/coursera-dl/coursera-dl/issues/205 ## Windows: Failed to create process -In `C:\Users\\AppData\Local\Programs\Python\Python35-32\Scripts` +In `C:\Users\\AppData\Local\Programs\Python\Python39-32\Scripts` or wherever Python installed (above is default for Windows) edit below file in idle: (right click on script name and select 'edit with idle in menu) @@ -510,13 +528,13 @@ coursera-dl-script from ``` -#!c:\users\\appdata\local\programs\python\python35-32\python.exe +#!c:\users\\appdata\local\programs\python\python39-32\python.exe ``` to ``` -#"!c:\users\\appdata\local\programs\python\python35-32\python.exe" +#"!c:\users\\appdata\local\programs\python\python39-32\python.exe" ``` (add quotes). This is a known pip bug. @@ -617,10 +635,9 @@ I enjoy getting feedback. Here are a few of the comments I've received: # Contact -Please, post bugs and issues on [github][11]. Send other comments to Rogério -Theodoro de Brito (the current maintainer): rbrito@ime.usp.br (twitter: -[@rtdbrito][21]) or to John Lehmann (the original author): first last at -geemail dotcom (twitter: [@jplehmann][12]). +Please, post bugs and issues on [github][11]. Please, **DON'T** send support +requests privately to the maintainers! We are quite swamped with day-to-day +activities. If you have problems, **PLEASE**, file them on the issue tracker. [1]: https://www.coursera.org [2]: https://sourceforge.net/projects/gnuwin32/files/wget/1.11.4-1/wget-1.11.4-1-setup.exe @@ -633,7 +650,6 @@ geemail dotcom (twitter: [@jplehmann][12]). [9]: https://chrome.google.com/webstore/detail/cookietxt-export/lopabhfecdfhgogdbojmaicoicjekelh [10]: https://addons.mozilla.org/en-US/firefox/addon/export-cookies/ [11]: https://github.com/coursera-dl/coursera-dl/issues -[12]: https://twitter.com/jplehmann [13]: http://techcrunch.com/2013/02/20/coursera-adds-29-schools-90-courses-and-4-new-languages-to-its-online-learning-platform/ [14]: http://www.tunapanda.org [15]: https://github.com/html5lib/html5lib-python @@ -642,7 +658,6 @@ geemail dotcom (twitter: [@jplehmann][12]). [18]: http://ww45.python-distribute.org/pip_distribute.png [19]: https://pypi.python.org/pypi/six/ [20]: https://www.coursera.org/about/terms -[21]: https://twitter.com/rtdbrito [22]: https://pypi.python.org/ [23]: https://pypi.python.org/pypi/coursera-dl [issue213]: https://github.com/coursera-dl/coursera-dl/issues/213 diff --git a/appveyor.yml b/appveyor.yml index f9ea741..2aab721 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -32,30 +32,6 @@ environment: PYTHON_VERSION: "2.7.x" # currently 2.7.11 PYTHON_ARCH: "64" - - PYTHON: "C:\\Python33" - PYTHON_VERSION: "3.3.x" # currently 3.3.5 - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python33-x64" - PYTHON_VERSION: "3.3.x" # currently 3.3.5 - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python34" - PYTHON_VERSION: "3.4.x" # currently 3.4.3 - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4.x" # currently 3.4.3 - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python35" - PYTHON_VERSION: "3.5.x" # currently 3.5.1 - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5.x" # currently 3.5.1 - PYTHON_ARCH: "64" - - PYTHON: "C:\\Python36" PYTHON_VERSION: "3.6.x" # currently 3.6.? PYTHON_ARCH: "32" @@ -64,6 +40,30 @@ environment: PYTHON_VERSION: "3.6.x" # currently 3.6.? PYTHON_ARCH: "64" + - PYTHON: "C:\\Python37" + PYTHON_VERSION: "3.7.x" # currently 3.7.? + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7.x" # currently 3.7.? + PYTHON_ARCH: "64" + + - PYTHON: "C:\\Python38" + PYTHON_VERSION: "3.8.x" # currently 3.8.? + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python38-x64" + PYTHON_VERSION: "3.8.x" # currently 3.8.? + PYTHON_ARCH: "64" + + - PYTHON: "C:\\Python39" + PYTHON_VERSION: "3.8.x" # currently 3.9.? + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python38-x64" + PYTHON_VERSION: "3.8.x" # currently 3.9.? + PYTHON_ARCH: "64" + init: - "ECHO %PYTHON%" - ps: "ls C:/Python*" diff --git a/coursera/__init__.py b/coursera/__init__.py index 8e747cc..eebcbfe 100644 --- a/coursera/__init__.py +++ b/coursera/__init__.py @@ -1 +1 @@ -__version__ = '0.11.4' +__version__ = '0.11.5' diff --git a/coursera/commandline.py b/coursera/commandline.py index 9047b0a..2fd80d7 100644 --- a/coursera/commandline.py +++ b/coursera/commandline.py @@ -341,6 +341,14 @@ def parse_args(args=None): group_adv_auth = parser.add_argument_group( 'Advanced authentication options') + group_adv_auth.add_argument( + '-ca', + '--cauth', + dest='cookies_cauth', + action='store', + default=None, + help='cauth cookie value from browser') + group_adv_auth.add_argument( '-c', '--cookies_file', @@ -492,7 +500,7 @@ def parse_args(args=None): logging.error('Cookies file not found: %s', args.cookies_file) sys.exit(1) - if not args.cookies_file: + if not args.cookies_file and not args.cookies_cauth: try: args.username, args.password = get_credentials( username=args.username, password=args.password, diff --git a/coursera/coursera_dl.py b/coursera/coursera_dl.py index 8b7c204..da18281 100644 --- a/coursera/coursera_dl.py +++ b/coursera/coursera_dl.py @@ -3,7 +3,7 @@ # Authors and copyright: # © 2012-2013, John Lehmann (first last at geemail dotcom or @jplehmann) -# © 2012-2015, Rogério Brito (r lastname at ime usp br) +# © 2012-2020, Rogério Theodoro de Brito # © 2013, Jonas De Taeye (first dt at fastmail fm) # # Contributions are welcome, but please add new unit tests to test your changes @@ -233,7 +233,10 @@ def main(): return session = get_session() - login(session, args.username, args.password) + if args.cookies_cauth: + session.cookies.set('CAUTH', args.cookies_cauth) + else: + login(session, args.username, args.password) if args.specialization: args.class_names = expand_specializations(session, args.class_names) diff --git a/requirements.txt b/requirements.txt index 6c56755..dc3ac2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ beautifulsoup4>=4.1.3 requests>=2.10.0 six>=1.5.0 -urllib3>=1.10 +urllib3>=1.23 pyasn1>=0.1.7 keyring>=4.0 configargparse>=0.12.0 diff --git a/setup.py b/setup.py index 9dffac9..a3da7fb 100644 --- a/setup.py +++ b/setup.py @@ -60,9 +60,8 @@ def read_file(filename, alt=None): generate_readme_rst() long_description = read_file( - 'README.rst', - 'Generate README.rst from README.md via pandoc!\n\nExample: ' - 'pandoc --from=markdown --to=rst --output=README.rst README.md' + 'README.md', + 'Cannot read README.md' ) requirements = read_file('requirements.txt') dev_requirements = read_file('requirements-dev.txt') @@ -89,7 +88,7 @@ setup( name='coursera-dl', version=__version__, maintainer='Rogério Theodoro de Brito', - maintainer_email='rbrito@ime.usp.br', + maintainer_email='rbrito@gmail.com', license='LGPL', url='https://github.com/coursera-dl/coursera-dl', @@ -101,6 +100,7 @@ setup( description='Script for downloading Coursera.org videos and naming them.', long_description=long_description, + long_description_content_type='text/markdown', keywords=['coursera-dl', 'coursera', 'download', 'education', 'MOOCs', 'video'], classifiers=trove_classifiers,