From 5283a40697686f94166b9dec4bc91b0a919f0af9 Mon Sep 17 00:00:00 2001 From: Mike Date: Sat, 27 Apr 2019 14:37:49 +0800 Subject: [PATCH 1/7] Create Dockerfile --- docker/Dockerfile | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 docker/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..f9054fad5 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,45 @@ +FROM alpine:3.9 as builder_pandoc +RUN apk update && \ + apk add cabal \ + zlib-dev \ + wget \ + ghc \ + musl-dev && \ + cabal update && \ + cabal install --upgrade-dependencies --enable-per-component -j --force-reinstalls pandoc + +FROM alpine:3.9 as builder_ytdl +COPY --from=builder_pandoc /root/.cabal /root/.cabal +RUN apk update && \ + apk add ffmpeg \ + rtmpdump \ + mplayer \ + mpv \ + python3 \ + git \ + make \ + zip && \ + ln -s /usr/bin/python3 /usr/bin/python && \ + ln -s /root/.cabal/bin/pandoc /usr/local/bin/pandoc && \ + git clone https://github.com/ytdl-org/youtube-dl.git && \ + cd /youtube-dl && \ + make -j && \ + make install + +FROM alpine:3.9 as final +COPY --from=builder_ytdl /usr/local/bin/youtube-dl /usr/local/bin/youtube-dl +COPY --from=builder_ytdl /usr/local/man/man1/youtube-dl.1 /usr/local/man/man1/youtube-dl.1 +RUN apk update && \ + apk add ffmpeg \ + rtmpdump \ + mplayer \ + mpv \ + python3 && \ + ln -s /usr/bin/python3 /usr/bin/python && \ + youtube-dl --version && \ + rm -rf /var/cache/apk/* + +COPY init /init +WORKDIR /home/dockeruser +ENTRYPOINT ["/init"] + From 94510116f63156a60212f2f7c3b6308f92a4f5aa Mon Sep 17 00:00:00 2001 From: Mike Date: Sat, 27 Apr 2019 14:40:03 +0800 Subject: [PATCH 2/7] Update Dockerfile --- docker/Dockerfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f9054fad5..86ad935a3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,5 @@ +# Builder container for pandoc, prerequisite for building youtube-dl +# (so build environment isn't in final container, to save space) FROM alpine:3.9 as builder_pandoc RUN apk update && \ apk add cabal \ @@ -8,6 +10,8 @@ RUN apk update && \ cabal update && \ cabal install --upgrade-dependencies --enable-per-component -j --force-reinstalls pandoc +# Builder container for youtube-dl +# (so build environment isn't in final container, to save space) FROM alpine:3.9 as builder_ytdl COPY --from=builder_pandoc /root/.cabal /root/.cabal RUN apk update && \ @@ -26,9 +30,12 @@ RUN apk update && \ make -j && \ make install +# Final container FROM alpine:3.9 as final +# Copy youtube-dl binary and manpage into container from builder container COPY --from=builder_ytdl /usr/local/bin/youtube-dl /usr/local/bin/youtube-dl COPY --from=builder_ytdl /usr/local/man/man1/youtube-dl.1 /usr/local/man/man1/youtube-dl.1 +# Install & configure prerequisites for youtube-dl RUN apk update && \ apk add ffmpeg \ rtmpdump \ @@ -38,8 +45,7 @@ RUN apk update && \ ln -s /usr/bin/python3 /usr/bin/python && \ youtube-dl --version && \ rm -rf /var/cache/apk/* - +# Copy init script, set workdir & entrypoint COPY init /init WORKDIR /home/dockeruser ENTRYPOINT ["/init"] - From 77573867f8a01bebca110bea3860693039d0d7e0 Mon Sep 17 00:00:00 2001 From: Mike Date: Sat, 27 Apr 2019 14:43:58 +0800 Subject: [PATCH 3/7] Create init --- docker/init | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docker/init diff --git a/docker/init b/docker/init new file mode 100644 index 000000000..eed39ad65 --- /dev/null +++ b/docker/init @@ -0,0 +1,16 @@ +#!/bin/sh + +# Create dockeruser user/group +addgroup -g $PGID dockeruser +adduser -D -u $PUID -G dockeruser -h /home/dockeruser dockeruser +chown $PUID:$PGID /home/dockeruser + +# Change ownership of youtube-dl and set sticky bit so youtube-dl runs as dockeruser (rootless operation) +chown $PUID:$PGID `which youtube-dl` +chmod u+s,g+s `which youtube-dl` + +# Ensure .netrc has correct permissions (if present) +chown root:root /root/.netrc + +# Run youtube-dl with remainder of command line arguments +youtube-dl $@ From 5aafc631502cf9fc984c9dd526feadf6ce6a68f1 Mon Sep 17 00:00:00 2001 From: Mike Date: Sat, 27 Apr 2019 15:00:56 +0800 Subject: [PATCH 4/7] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 86ad935a3..57af02d3c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -47,5 +47,5 @@ RUN apk update && \ rm -rf /var/cache/apk/* # Copy init script, set workdir & entrypoint COPY init /init -WORKDIR /home/dockeruser +WORKDIR /workdir ENTRYPOINT ["/init"] From 26fd6011f6c09b65034854e7338a84a5f71ff918 Mon Sep 17 00:00:00 2001 From: Mike Date: Sat, 27 Apr 2019 15:54:18 +0800 Subject: [PATCH 5/7] Create README.md ``` Where `CONTAINER` is the name of the running container. To start a container with a shell (instead of `youtube-dl`), execu --- docker/README.md | 186 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 docker/README.md diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..9a71aea66 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,186 @@ +## youtube-dl +youtube-dl - download videos from youtube.com or other video platforms + +## Quick Start +**NOTE**: The docker command provided in this quick start is given as an example, and parameters should be adjusted to your needs. + +It is suggested to configure an alias as follows (and place into your `.bash_aliases` file): + +``` +alias youtube-dl='docker run \ + --rm -i \ + -e PGID=$(id -g) \ + -e PUID=$(id -u) \ + -v $(pwd):/workdir:rw \ + ytdl-org/youtube-dl' +``` + +When run (eg: `youtube-dl https://www.youtube.com/watch?v=dQw4w9WgXcQ`), it will download the video to the current working directory, and take any command line arguments that the normal youtube-dl binary would. + +## Using a config file + +To prevent having to specify many command line arguments every time you run youtube-dl, you may wish to have an external configuation file. + +In order for the docker container to use the configuration file, it must be mapped through to the container. + +``` +docker run \ + --rm -i \ + -e PGID=$(id -g) \ + -e PUID=$(id -u) \ + -v /path/to/downloaded/videos:/workdir:rw \ + -v /path/to/youtube-dl.conf:/etc/youtube-dl.conf:ro \ + ytdl-org/youtube-dl +``` + +Where: +* `/path/to/downloaded/videos` is where youtube-dl will download videos to (use `$(pwd)` to downloade to current working directory. +* `/path/to/youtube-dl.conf` is the path to your youtube-dl.conf file. + +## Environment Variables + +To customize some properties of the container, the following environment variables can be passed via the `-e` parameter (one for each variable). This paramater has the format `=`. + +| Variable | Description | Recommended Setting | +|----------|-------------|---------------------| +| PGID | The Group ID that the `youtube-dl` process will run as | `$(id -u)` for the current user's GID | +| PUID | The User ID that the `youtube-dl` process will run as | `$(id -g)` for the current user's UID | + +## Authentication using .netrc + +If you want to download videos that require authentication (or your youtube subscriptions for example, see below), it is recommended to use a .netrc file. + +You can create a file with the following syntax: + +``` +machine youtube login USERNAME password PASSWORD +``` + +Where: +* `USERNAME` is replaced with your youtube account username +* `PASSWORD` is replaced with your youtube account password + +You may need to disable some account security settings (such as '2-Step Verification' and 'Use your phone to sign in', so it is suggested to make a long, complex password (eg: 32 random characters). + +This file can then be mapped through to the container as a .netrc file, eg: + +``` +docker run \ + --rm -i \ + -e PGID=$(id -g) \ + -e PUID=$(id -u) \ + -v /path/to/downloaded/videos:/workdir:rw \ + -v /path/to/youtube-dl.conf:/etc/youtube-dl.conf:ro \ + -v /path/to/netrc_file:/home/dockeruser/.netrc:ro + ytdl-org/youtube-dl +``` + +## Data Volumes + +There are no data volumes explicity set in the Dockerfile, however: + +| Container Path | Permissions | Description | +|----------------|-------------|-------------| +| `/workdir` | rw | The `youtube-dl` process is executed with a working directory of `/workdir`. Thus, unless you override the output directory with the `--output` argument on the command line or via a configuration file, videos will end up in this directory. | +| `/etc/youtube-dl.conf` | ro | The `youtube-dl` process will look in this location for a configuration file by default. | +| `/home/dockeruser/.netrc` | ro | The `youtube-dl` process will look in this location for a .netrc file (if `--netrc` is specified on the command line or via a configuration file). | + +## Scheduled download of youtube subscriptions + +In order to perform a scheduled download of youtube subscriptions, it is recommended to use the following command to be executed via cron on a regular basis (eg: daily). + +``` +docker run \ + --rm + -i + --name youtube-dl-cron + -e PGID=GID + -e PUID=UID + --cpus CPUS + -v /path/to/netrc:/home/dockeruser/.netrc:ro + -v /path/to/youtube/subscriptions:/workdir:rw + -v /path/to/youtube-dl.conf:/etc/youtube-dl.conf:ro + mikenye/youtube-dl \ + :ytsubscriptions \ + --dateafter now-5days \ + --download-archive /workdir/.youtube-dl-archive \ + --cookies /workdir/cookiejar.dat \ + --netrc \ + --limit-rate 5000 2>> /path/to/logs/youtube-dl.err >> /path/to/logs/youtube-dl.log +``` + +Where: +* `GID`: a group ID to run as (if in a normal user's crontab, use `$(id -g)` +* `UID`: a user ID to run as (if in a normal user's crontab, use `$(id -u)` +* `CPUS`: the number of CPUs to constrain the docker container to. This may be required to prevent impacting system performance if transcoding takes place. If not, the `--cpus` argument can be completely omitted. +* `/path/to/netrc`: the path to the file containing your youtube credentials, see above. +* `/path/to/youtube/subscriptions`: the path where videos will be saved. +* `/path/to/youtube-dl.conf`: the path to your `youtube-dl.conf` file, where settings such as output file naming, quality, etc can be determined. +* `/path/to/logs/youtube-dl.err`: the path to the error log, if desired +* `/path/to/logs/youtube-dl.log`: the path to the application log, if desired + +Notes: +* `--rm` is given so the container is destroyed when execution is finished. This will prevent your drive from slowly filling up with exited containers. +* `--name youtube-dl-cron` is given so that multiple instances are not started by cron. In the event the previous container is still running, docker will simply exit with an error that the container name is already taken. +* `--dateafter now-5days` is given to limit youtube-dl to only download recent videos. Feel free to adjust as required. +* `--limit-rate` is given to prevent impacting your internet connection. Feel free to adjust as required. + +In the example above, a configuration file is used. This allows us to easily add commands to select a specific quality, and name the videos with a specific format. For example, to download the videos into a format recognised by Plex, you could use the following: + +``` +-v +--ignore-errors +--no-overwrites +--continue +--no-post-overwrites +--add-metadata +--write-thumbnail +--playlist-reverse +--write-description +--write-info-json +--write-annotations +--format "best[height<=?1080]" +--fixup fix +--output '/workdir/%(uploader)s/%(uploader)s - %(upload_date)s - %(title)s - %(id)s.%(ext)s' +``` + +The above example config file will: +* Ignore errors +* Will not overwrite existing videos +* Will continue downloading in the event a download is interrupted +* Will download metadata and embed into the resulting video, so that Plex will be able to display this information +* Will download oldest videos first, so videos can be sorted by "date added" in Plex +* Will download the best quality up to a maximum of 1080p (prevent 4K downloads) +* Will format videos with a separate folder for each uploader. + +## Ports + +No port mappings are required for this container. + +## Docker Image Update + +If the system on which the container runs doesn't provide a way to easily update the Docker image (eg: watchtower), simply pull the latest version of the container: + +``` +docker pull ytdl-org/youtube-dl +```` + +## Shell access + +To get shell access to a running container, execute the following command: + +``` +docker exec -ti CONTAINER sh +``` + +Where `CONTAINER` is the name of the running container. + +To start a container with a shell (instead of `youtube-dl`), execute the following command: + +``` +docker run --rm -ti --entrypoint=/bin/sh ytdl-org/youtube-dl +``` + +## Support or Contact + +Having troubles with the container or have questions? Please create a new issue on our github. From 1b35543bc85e34fceeca8568d9751675199ee3a4 Mon Sep 17 00:00:00 2001 From: mikenye Date: Sat, 27 Apr 2019 19:24:48 +0800 Subject: [PATCH 6/7] modified: Dockerfile new file: build_and_push.sh modified: init Script build & tag process. Make youtube-dl run as regular user. --- docker/Dockerfile | 7 ++++--- docker/build_and_push.sh | 42 ++++++++++++++++++++++++++++++++++++++++ docker/init | 25 ++++++++++++------------ 3 files changed, 59 insertions(+), 15 deletions(-) create mode 100755 docker/build_and_push.sh mode change 100644 => 100755 docker/init diff --git a/docker/Dockerfile b/docker/Dockerfile index 57af02d3c..635d30474 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -35,16 +35,17 @@ FROM alpine:3.9 as final # Copy youtube-dl binary and manpage into container from builder container COPY --from=builder_ytdl /usr/local/bin/youtube-dl /usr/local/bin/youtube-dl COPY --from=builder_ytdl /usr/local/man/man1/youtube-dl.1 /usr/local/man/man1/youtube-dl.1 -# Install & configure prerequisites for youtube-dl +# Install & configure s6 overlay, then prerequisites for youtube-dl RUN apk update && \ apk add ffmpeg \ rtmpdump \ mplayer \ mpv \ - python3 && \ + python3 \ + bash && \ ln -s /usr/bin/python3 /usr/bin/python && \ youtube-dl --version && \ - rm -rf /var/cache/apk/* + rm -rf /var/cache/apk/* /tmp/* # Copy init script, set workdir & entrypoint COPY init /init WORKDIR /workdir diff --git a/docker/build_and_push.sh b/docker/build_and_push.sh new file mode 100755 index 000000000..96fe93265 --- /dev/null +++ b/docker/build_and_push.sh @@ -0,0 +1,42 @@ +#!/bin/bash -x + +# Build "latest" +docker build . -t ytdl-org/youtube-dl:latest +build_exit=$? + + +# Get version of latest container +# Repeat as running straight after the build can give a 'bad interpreter: Text file busy' error +n=0 +until [ $n -ge 5 ] +do + build_version=$(docker run --rm -ti ytdl-org/youtube-dl --version) + if [ $? -eq 0 ]; then + build_version=$(echo $build_version | sed 's/\r$//') + break + fi + n=$[$n+1] + sleep 15 +done +if [ $n -ge 5 ]; then + echo "Failed when trying to get youtube-dl version in latest container :(" + exit 1 +fi + +# If the build was successful, then we can tag with current version +if [ $build_exit -eq 0 ]; then + docker tag ytdl-org/youtube-dl:latest ytdl-org/youtube-dl:$build_version + tag_exit=$? +fi + +if [ $build_exit -eq 0 ]; then + if [ $tag_exit -eq 0 ]; then + # If building and tagging was successful, then push + docker push ytdl-org/youtube-dl:latest + docker push ytdl-org/youtube-dl:$build_version + exit 0 + fi +fi + +echo "Something went wrong..." +exit 1 diff --git a/docker/init b/docker/init old mode 100644 new mode 100755 index eed39ad65..2fe9012bd --- a/docker/init +++ b/docker/init @@ -1,16 +1,17 @@ -#!/bin/sh +#!/bin/bash + +export YOUTUBEDLARGS=$@ + +YOUTUBEDLPGID=${PGID:-1000} +YOUTUBEDLPUID=${PUID:-1000} # Create dockeruser user/group -addgroup -g $PGID dockeruser -adduser -D -u $PUID -G dockeruser -h /home/dockeruser dockeruser -chown $PUID:$PGID /home/dockeruser - -# Change ownership of youtube-dl and set sticky bit so youtube-dl runs as dockeruser (rootless operation) -chown $PUID:$PGID `which youtube-dl` -chmod u+s,g+s `which youtube-dl` - -# Ensure .netrc has correct permissions (if present) -chown root:root /root/.netrc +addgroup -g $YOUTUBEDLPGID dockeruser +adduser -D -u $YOUTUBEDLPUID -G dockeruser -h /home/dockeruser dockeruser +chown $YOUTUBEDLPUID:$YOUTUBEDLPGID /home/dockeruser +export HOME=/home/dockeruser # Run youtube-dl with remainder of command line arguments -youtube-dl $@ +su -m dockeruser <<'EOF' +youtube-dl $YOUTUBEDLARGS +EOF From d8e39aca340be1002375d8c241aa219be3781ff3 Mon Sep 17 00:00:00 2001 From: Mike Date: Sat, 27 Apr 2019 19:31:51 +0800 Subject: [PATCH 7/7] Update README.md --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 9a71aea66..119126e62 100644 --- a/docker/README.md +++ b/docker/README.md @@ -104,7 +104,7 @@ docker run \ :ytsubscriptions \ --dateafter now-5days \ --download-archive /workdir/.youtube-dl-archive \ - --cookies /workdir/cookiejar.dat \ + --cookies /workdir/.youtube-dl-cookiejar \ --netrc \ --limit-rate 5000 2>> /path/to/logs/youtube-dl.err >> /path/to/logs/youtube-dl.log ```