Compare commits
No commits in common. "main" and "v0.5.4" have entirely different histories.
1
.github/FUNDING.yml
vendored
1
.github/FUNDING.yml
vendored
@ -1 +0,0 @@
|
|||||||
github: tardisx
|
|
||||||
2
.github/workflows/go.yml
vendored
2
.github/workflows/go.yml
vendored
@ -16,7 +16,7 @@ jobs:
|
|||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: actions/setup-go@v2
|
uses: actions/setup-go@v2
|
||||||
with:
|
with:
|
||||||
go-version: 1.22
|
go-version: 1.17
|
||||||
|
|
||||||
- name: Build
|
- name: Build
|
||||||
run: go build -v ./...
|
run: go build -v ./...
|
||||||
|
|||||||
5
.gitignore
vendored
5
.gitignore
vendored
@ -1,6 +1,3 @@
|
|||||||
gropple
|
gropple
|
||||||
release
|
|
||||||
dist
|
dist
|
||||||
.env
|
release
|
||||||
dist/
|
|
||||||
.DS_Store
|
|
||||||
|
|||||||
@ -1,53 +0,0 @@
|
|||||||
version: 2
|
|
||||||
|
|
||||||
before:
|
|
||||||
hooks:
|
|
||||||
- go mod tidy
|
|
||||||
- go test ./...
|
|
||||||
|
|
||||||
builds:
|
|
||||||
- env:
|
|
||||||
- CGO_ENABLED=0
|
|
||||||
goos:
|
|
||||||
- linux
|
|
||||||
- windows
|
|
||||||
- darwin
|
|
||||||
|
|
||||||
archives:
|
|
||||||
- formats: [tar.gz]
|
|
||||||
# this name template makes the OS and Arch compatible with the results of `uname`.
|
|
||||||
name_template: >-
|
|
||||||
{{ .ProjectName }}_
|
|
||||||
{{- title .Os }}_
|
|
||||||
{{- if eq .Arch "amd64" }}x86_64
|
|
||||||
{{- else if eq .Arch "386" }}i386
|
|
||||||
{{- else }}{{ .Arch }}{{ end }}
|
|
||||||
{{- if .Arm }}v{{ .Arm }}{{ end }}
|
|
||||||
# use zip for windows archives
|
|
||||||
format_overrides:
|
|
||||||
- goos: windows
|
|
||||||
formats: [zip]
|
|
||||||
|
|
||||||
changelog:
|
|
||||||
disable: true
|
|
||||||
|
|
||||||
dockers:
|
|
||||||
- image_templates:
|
|
||||||
- "tardisx/gropple:{{ .Tag }}-amd64"
|
|
||||||
use: buildx
|
|
||||||
build_flag_templates:
|
|
||||||
- "--pull"
|
|
||||||
- "--platform=linux/amd64"
|
|
||||||
- image_templates:
|
|
||||||
- "tardisx/gropple:{{ .Tag }}-arm64"
|
|
||||||
use: buildx
|
|
||||||
build_flag_templates:
|
|
||||||
- "--pull"
|
|
||||||
- "--platform=linux/arm64"
|
|
||||||
goarch: arm64
|
|
||||||
|
|
||||||
docker_manifests:
|
|
||||||
- name_template: "tardisx/gropple:{{ .Tag }}"
|
|
||||||
image_templates:
|
|
||||||
- "tardisx/gropple:{{ .Tag }}-amd64"
|
|
||||||
- "tardisx/gropple:{{ .Tag }}-arm64"
|
|
||||||
15
.vscode/launch.json
vendored
15
.vscode/launch.json
vendored
@ -1,15 +0,0 @@
|
|||||||
{
|
|
||||||
// Use IntelliSense to learn about possible attributes.
|
|
||||||
// Hover to view descriptions of existing attributes.
|
|
||||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
|
||||||
"version": "0.2.0",
|
|
||||||
"configurations": [
|
|
||||||
{
|
|
||||||
"name": "Launch Package",
|
|
||||||
"type": "go",
|
|
||||||
"request": "launch",
|
|
||||||
"mode": "auto",
|
|
||||||
"program": "${fileDirname}"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
5
.vscode/settings.json
vendored
5
.vscode/settings.json
vendored
@ -1,12 +1,9 @@
|
|||||||
{
|
{
|
||||||
"cSpell.words": [
|
"cSpell.words": [
|
||||||
"Cleanup",
|
"Cleanup",
|
||||||
"frag",
|
|
||||||
"gropple",
|
|
||||||
"succ",
|
|
||||||
"tmpl",
|
"tmpl",
|
||||||
"vars",
|
"vars",
|
||||||
"youtube"
|
"gropple"
|
||||||
],
|
],
|
||||||
"cSpell.language": "en-GB"
|
"cSpell.language": "en-GB"
|
||||||
}
|
}
|
||||||
54
CHANGELOG.md
54
CHANGELOG.md
@ -2,59 +2,7 @@
|
|||||||
|
|
||||||
All notable changes to this project will be documented in this file.
|
All notable changes to this project will be documented in this file.
|
||||||
|
|
||||||
## [v1.1.4] - 2025-04-25
|
## [Unreleased]
|
||||||
|
|
||||||
- Minor refactorings, upgrade dependencies
|
|
||||||
- Migrate to goreleaser v2 configuration
|
|
||||||
- Generate arm64 docker builds
|
|
||||||
|
|
||||||
## [v1.1.3] - 2024-03-17
|
|
||||||
|
|
||||||
- Code cleanups, better error checking
|
|
||||||
|
|
||||||
## [v1.1.2] - 2024-03-16
|
|
||||||
|
|
||||||
- Fix a crash for a certain pattern of log line
|
|
||||||
|
|
||||||
## [v1.1.1] - 2023-12-08
|
|
||||||
|
|
||||||
- Fix bug where a brand-new config was created with an out-of-date version
|
|
||||||
- Fix for portable mode and using executable in the current working directory
|
|
||||||
|
|
||||||
## [v1.1.0] - 2023-11-25
|
|
||||||
|
|
||||||
- Add feature to bulk add URL's for downloading
|
|
||||||
|
|
||||||
## [v1.0.1] - 2023-11-24
|
|
||||||
|
|
||||||
- Fix crash on migrating a config that had > 1 destinations
|
|
||||||
|
|
||||||
## [v1.0.0] - 2023-11-23
|
|
||||||
|
|
||||||
- Don't start downloads until "start download" is pressed
|
|
||||||
- Add "download option" for more per-download customisability, especially for destinations
|
|
||||||
- Removed "destinations" as that is now possible more flexibly with download options.
|
|
||||||
- Existing configurations using destinations are automatically migrated to an appropriate `yt-dlp -o ...` download options
|
|
||||||
- Gropple now available via docker
|
|
||||||
- Clean up web interface display on index page, especially when a playlist with many files is downloading
|
|
||||||
|
|
||||||
## [v0.6.0] - 2023-03-15
|
|
||||||
|
|
||||||
- Configurable destinations for downloads
|
|
||||||
- Multiple destination directories can be configured
|
|
||||||
- When queueing a download, an alternate destination can be selected
|
|
||||||
- When downloading from a playlist, show the total number of videos and how many have been downloaded
|
|
||||||
- Show version in web UI
|
|
||||||
- Improve index page (show URL of queued downloads instead of nothing)
|
|
||||||
- Fixes and improvements to capturing output info and showing it in the UI
|
|
||||||
- Show all log output in the popup
|
|
||||||
- Fixes to handling of queued downloads
|
|
||||||
- Fix portable mode to look in binary directory, not current directory
|
|
||||||
- Automatically cleanup download list, removing old entries automatically
|
|
||||||
|
|
||||||
## [v0.5.5] - 2022-04-09
|
|
||||||
|
|
||||||
- Fix a bug which would erase configuration when migrating from v1 to v2 config
|
|
||||||
|
|
||||||
## [v0.5.4] - 2022-04-07
|
## [v0.5.4] - 2022-04-07
|
||||||
|
|
||||||
|
|||||||
@ -1,9 +0,0 @@
|
|||||||
FROM ubuntu:noble
|
|
||||||
COPY gropple /
|
|
||||||
|
|
||||||
RUN apt update && apt install -y curl python3 ffmpeg
|
|
||||||
RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/bin/yt-dlp
|
|
||||||
RUN chmod a+x /usr/bin/yt-dlp
|
|
||||||
|
|
||||||
# Run executable
|
|
||||||
CMD ["/gropple", "--config-path", "/config/gropple.json"]
|
|
||||||
203
README.md
203
README.md
@ -1,196 +1,97 @@
|
|||||||
# gropple
|
# gropple
|
||||||
|
|
||||||
A frontend to youtube-dl (or compatible forks, like yt-dlp) to download videos
|
A frontend to youtube-dl (or compatible forks, like yt-dlp) to download videos with a single click, straight from your web browser.
|
||||||
with a single click, straight from your web browser.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## Installing
|
## Pre-requisites
|
||||||
|
|
||||||
### From Source
|
* some familiarity with the command line
|
||||||
|
* youtube-dl (plus any of its required dependencies, like ffmpeg)
|
||||||
|
* golang compiler (only if you'd like to build from source)
|
||||||
|
|
||||||
|
## Build
|
||||||
|
|
||||||
go build
|
go build
|
||||||
|
|
||||||
### Standalone Binaries
|
## Binaries
|
||||||
|
|
||||||
Binaries are available at <https://github.com/tardisx/gropple/releases> for most
|
Binaries are available at https://github.com/tardisx/gropple/releases
|
||||||
platforms.
|
|
||||||
|
Gropple will automatically check for available updates and prompt you to upgrade.
|
||||||
|
|
||||||
## Running
|
## Running
|
||||||
|
|
||||||
### From Binaries
|
./gropple
|
||||||
|
|
||||||
./gropple
|
|
||||||
|
|
||||||
There are no command line arguments. All configuration is done via the web
|
There are no command line arguments. All configuration is done via the web
|
||||||
interface. The address will be printed after startup:
|
interface. The address will be printed after startup:
|
||||||
|
|
||||||
2023/11/22 22:42:06 Starting gropple v1.0.0 - https://github.com/tardisx/gropple
|
2021/09/30 23:53:00 starting gropple v0.5.0 - https://github.com/tardisx/gropple
|
||||||
2023/11/22 22:42:07 Configuration loaded from /Users/username/path/config.yml
|
2021/09/30 23:53:00 go to http://localhost:6123 for details on installing the bookmarklet and to check status
|
||||||
2023/11/22 22:42:07 Visit http://localhost:6123 for details on installing the bookmarklet and to check status
|
|
||||||
|
|
||||||
### Docker
|
|
||||||
|
|
||||||
Copy the `docker-compose.yml` to a directory somewhere.
|
|
||||||
|
|
||||||
Edit the two `volume` entries to point to local paths where you would like to
|
|
||||||
store the config file, and the downloads (the path on the left hand side of the
|
|
||||||
colon).
|
|
||||||
|
|
||||||
Run `docker-compose up -d` to start the program.
|
|
||||||
|
|
||||||
Note that the docker images include `yt-dlp` and `ffmpeg` and are thus
|
|
||||||
completely self-contained.
|
|
||||||
|
|
||||||
Run `docker-compose logs` to see the output of the program, if you are having
|
|
||||||
problems.
|
|
||||||
|
|
||||||
## Using
|
## Using
|
||||||
|
|
||||||
Bring up `http://localhost:6283` (or the appropriate host if you are running it
|
Bring up `http://localhost:6283` (or your configured address) in your browser. You
|
||||||
on a different machine) in your browser. You should see a link to the
|
should see a link to the bookmarklet at the top of the screen, and the list of
|
||||||
bookmarklet at the top of the screen, and the list of downloads (currently
|
downloads (currently empty).
|
||||||
empty).
|
|
||||||
|
|
||||||
Drag the bookmarklet to your favourites bar, or otherwise bookmark it as you see
|
Drag the bookmarklet to your favourites bar, or otherwise bookmark it as you
|
||||||
fit. Any kind of browser bookmark should work. The bookmarklet contains embedded
|
see fit. Any kind of browser bookmark should work. The bookmarklet contains
|
||||||
javascript to pass the URL of whatever page you are currently on back to
|
embedded javascript to pass the URL of whatever page you are currently on back
|
||||||
gropple.
|
to gropple.
|
||||||
|
|
||||||
Whenever you are on a page with a video you would like to download just click
|
So, whenever you are on a page with a video you would like to download just
|
||||||
the bookmarklet.
|
click the bookmarklet.
|
||||||
|
|
||||||
A popup window will appear. Choose a download profile and the download will
|
A popup window will appear. Choose a download profile and the download will start.
|
||||||
start. The status will be shown in the window, updating in real time.
|
The status will be shown in the window, updating in real time.
|
||||||
|
|
||||||
There is also an optional "download option" you can choose. These are discussed
|
|
||||||
below.
|
|
||||||
|
|
||||||
You may close this window at any time without stopping the download, the status
|
You may close this window at any time without stopping the download, the status
|
||||||
of all downloads is available on the index page. Clicking on the id number will
|
of all downloads is available on the index page.
|
||||||
show the popup again.
|
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
Click the "config" link on the index page to configure gropple.
|
Click the "config" link on the index page to configure gropple. The default options
|
||||||
|
are fine if you are running on your local machine. If you are running it remotely
|
||||||
|
you will need to set the "server address" to ensure the bookmarklet has the correct
|
||||||
|
URL in it.
|
||||||
|
|
||||||
The options in each part are dicussed below.
|
### Configuring Downloaders
|
||||||
|
|
||||||
### Server
|
Gropple's default configuration uses the original youtube-dl and has two profiles set
|
||||||
|
up, one for downloading video, the other for downloading audio (mp3).
|
||||||
|
|
||||||
#### Port and Server Address
|
Note that gropple does not include any downloaders, you have to install them separately.
|
||||||
|
|
||||||
You can configure the port number here if you do not want the default of `6123`.
|
If you would like to use a youtube-dl fork (like [yt-dlp](https://github.com/yt-dlp/yt-dlp))
|
||||||
|
or change the options, you can do so on the right hand side. Create as many profiles as you
|
||||||
|
wish, whenever you start a download you can choose the appropriate profile.
|
||||||
|
|
||||||
If you are running it on a machine other than `localhost` you will need to set
|
Note that the command arguments must each be specified separately - see the default configuration
|
||||||
the "server address" to ensure the bookmarklet has the correct URL in it.
|
for an example.
|
||||||
Similarly, if you are running it behind a reverse proxy, the address here must
|
|
||||||
match what you would type in the browser so that the bookmarklet will work
|
|
||||||
correctly.
|
|
||||||
|
|
||||||
#### Download path
|
While gropple will use your `PATH` to find the executable, you can also specify a full path
|
||||||
|
instead. Note that any tools that the downloader calls itself (for instance, ffmpeg) will
|
||||||
The download path specifies where downloads will end up, *if* no specific `-o`
|
probably need to be available on your path.
|
||||||
options are passed to `yt-dlp`.
|
|
||||||
|
|
||||||
#### Maximum active downloads per domain
|
|
||||||
|
|
||||||
Gropple will limit the number of downloads per domain to this number. Increasing
|
|
||||||
this will likely result in failed downloads when server rate limiters notice
|
|
||||||
you.
|
|
||||||
|
|
||||||
#### UI popup size
|
|
||||||
|
|
||||||
Changes the size of the popup window.
|
|
||||||
|
|
||||||
### Download Profiles
|
|
||||||
|
|
||||||
Gropple's default configuration uses `yt-dlp` and has two profiles set up, one
|
|
||||||
for downloading video, the other for downloading audio (mp3).
|
|
||||||
|
|
||||||
Each download profile consists of a name (for your reference), a command to run,
|
|
||||||
and a number of arguments.
|
|
||||||
|
|
||||||
Note that gropple does not include any downloaders, you have to install them
|
|
||||||
separately (unless using the docker image).
|
|
||||||
|
|
||||||
If you would like to use a youtube-dl compatible fork or change the options you
|
|
||||||
can do so here. Create as many profiles as you wish, whenever you start a
|
|
||||||
download you can choose the appropriate profile.
|
|
||||||
|
|
||||||
Note that the command arguments must each be specified separately - see the
|
|
||||||
default configuration. For example, if you have a single argument like
|
|
||||||
`--audio-format mp3`, it will be parsed by the `yt-dlp` as a single, long
|
|
||||||
unknown argument, and will fail. This needs to be configured as two arguments,
|
|
||||||
`--audio-format` and `mp3`.
|
|
||||||
|
|
||||||
While gropple will use your `PATH` to find the executable, you can also specify
|
|
||||||
a full path instead. Note that any tools that the downloader calls itself (for
|
|
||||||
instance, `ffmpeg`) will need to be available on your path.
|
|
||||||
|
|
||||||
### Download Options
|
|
||||||
|
|
||||||
There are also an arbitrary amount of Download Options you can configure. Each
|
|
||||||
one specifies one or more extra arguments to add to the downloader command line.
|
|
||||||
The most common use for this is to have customised download paths. For instance,
|
|
||||||
sometimes you might want to bundle all files into a single directory, other
|
|
||||||
times you might want to separate files by download playlist URL or similar.
|
|
||||||
|
|
||||||
Most of this is done directly through appropriate options for `yt-dlp`, see the
|
|
||||||
[output template
|
|
||||||
documentation](https://github.com/yt-dlp/yt-dlp#output-template).
|
|
||||||
|
|
||||||
However, gropple offers two extra substitutions:
|
|
||||||
|
|
||||||
* `%GROPPLE_HOST%`
|
|
||||||
* `%GROPPLE_PATH%`
|
|
||||||
|
|
||||||
These will be replaced with the hostname, and path of the download,
|
|
||||||
respectively.
|
|
||||||
|
|
||||||
So, a playlist URL `https://www.youtube.com/@UsernameHere`
|
|
||||||
|
|
||||||
With a download option setup like this:
|
|
||||||
|
|
||||||
* Name of Option: Split by Host and Path
|
|
||||||
* Arguments:
|
|
||||||
* -o
|
|
||||||
* /Downloads/%GROPPLE_HOST%/%GROPPLE_PATH%/%(title)s [%(id)s].%(ext)s
|
|
||||||
|
|
||||||
Will result in downloads going into the path
|
|
||||||
`/Downloads/www.youtube.com/@UsernameHere/...`.
|
|
||||||
|
|
||||||
Note that this also means that `yt-dlp` can resume partially downloaded files, and
|
|
||||||
also automatically 'backfill', downloading only files that have not been
|
|
||||||
downloaded yet from that playlist.
|
|
||||||
|
|
||||||
## Downloading a list of URL's in bulk
|
|
||||||
|
|
||||||
From main index page you can click the "Bulk" link in the menu to bring up the
|
|
||||||
bulk queue page.
|
|
||||||
|
|
||||||
In all respects this acts the same as the usual bookmarklet, but it has a
|
|
||||||
textbox for pasting many URLs at once. All downloads will be queued immediately.
|
|
||||||
|
|
||||||
## Portable mode
|
## Portable mode
|
||||||
|
|
||||||
If you'd like to use gropple from a USB stick or similar, copy the config file
|
If you'd like to use gropple from a USB stick or similar, copy the config file from
|
||||||
from its default location (shown when you start gropple) to the same location as
|
it's default location (shown when you start gropple) to the same location as the binary, and rename it to `gropple.yml`.
|
||||||
the binary, and rename it to `gropple.yml`.
|
|
||||||
|
If that file is present in the same directory as the binary, it will be used instead.
|
||||||
|
|
||||||
## Problems
|
## Problems
|
||||||
|
|
||||||
Many download problems are diagnosable via the log - check in the popup window
|
Most download problems are probably diagnosable via the log - check in the popup window and scroll
|
||||||
and scroll the log down to the bottom. The most common problem is that `yt-dlp`
|
the log down to the bottom. The most common problem is that youtube-dl cannot be found, or its
|
||||||
cannot be found, or its dependency (like `ffmpeg`) cannot be found on your path.
|
dependency (like ffmpeg) cannot be found on your path.
|
||||||
|
|
||||||
Gropple only calls external tools like `yt-dlp` to do the downloading. If you
|
|
||||||
are having problems downloading from a site, make sure that `yt-dlp` is updated
|
|
||||||
to the latest version (`yd-dlp -U`).
|
|
||||||
|
|
||||||
For other problems, please file an issue on github.
|
For other problems, please file an issue on github.
|
||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
|
|
||||||
Many things. Please raise an issue after checking the [currently open
|
Many things. Please raise an issue after checking the [currently open issues](https://github.com/tardisx/gropple/issues).
|
||||||
issues](https://github.com/tardisx/gropple/issues).
|
|
||||||
|
|
||||||
|
|||||||
40
build_release.pl
Executable file
40
build_release.pl
Executable file
@ -0,0 +1,40 @@
|
|||||||
|
#!/usr/bin/env perl
|
||||||
|
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
|
||||||
|
open my $fh, "<", "main.go" || die $!;
|
||||||
|
|
||||||
|
my $version;
|
||||||
|
while (<$fh>) {
|
||||||
|
# CurrentVersion: "v0.04"
|
||||||
|
$version = $1 if /CurrentVersion:\s*"(v[\d\.]+)"/;
|
||||||
|
}
|
||||||
|
close $fh;
|
||||||
|
|
||||||
|
die "no version?" unless defined $version;
|
||||||
|
|
||||||
|
# quit if tests fail
|
||||||
|
system("go test ./...") && die "not building release with failing tests";
|
||||||
|
|
||||||
|
# so lazy
|
||||||
|
system "rm", "-rf", "release", "dist";
|
||||||
|
system "mkdir", "release";
|
||||||
|
system "mkdir", "dist";
|
||||||
|
|
||||||
|
my %build = (
|
||||||
|
win => { env => { GOOS => 'windows', GOARCH => 'amd64' }, filename => 'gropple.exe' },
|
||||||
|
linux => { env => { GOOS => 'linux', GOARCH => 'amd64' }, filename => 'gropple' },
|
||||||
|
mac => { env => { GOOS => 'darwin', GOARCH => 'amd64' }, filename => 'gropple' },
|
||||||
|
);
|
||||||
|
|
||||||
|
foreach my $type (keys %build) {
|
||||||
|
mkdir "release/$type";
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach my $type (keys %build) {
|
||||||
|
local $ENV{GOOS} = $build{$type}->{env}->{GOOS};
|
||||||
|
local $ENV{GOARCH} = $build{$type}->{env}->{GOARCH};
|
||||||
|
system "go", "build", "-o", "release/$type/" . $build{$type}->{filename};
|
||||||
|
system "zip", "-j", "dist/gropple-$type-$version.zip", ( glob "release/$type/*" );
|
||||||
|
}
|
||||||
134
config/config.go
134
config/config.go
@ -7,7 +7,6 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gopkg.in/yaml.v2"
|
"gopkg.in/yaml.v2"
|
||||||
@ -20,39 +19,22 @@ type Server struct {
|
|||||||
MaximumActiveDownloads int `yaml:"maximum_active_downloads_per_domain" json:"maximum_active_downloads_per_domain"`
|
MaximumActiveDownloads int `yaml:"maximum_active_downloads_per_domain" json:"maximum_active_downloads_per_domain"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DownloadProfile holds the details for executing a downloader
|
|
||||||
type DownloadProfile struct {
|
type DownloadProfile struct {
|
||||||
Name string `yaml:"name" json:"name"`
|
Name string `yaml:"name" json:"name"`
|
||||||
Command string `yaml:"command" json:"command"`
|
Command string `yaml:"command" json:"command"`
|
||||||
Args []string `yaml:"args" json:"args"`
|
Args []string `yaml:"args" json:"args"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DownloadOption contains configuration for extra arguments to pass to the download command
|
|
||||||
type DownloadOption struct {
|
|
||||||
Name string `yaml:"name" json:"name"`
|
|
||||||
Args []string `yaml:"args" json:"args"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// UI holds the configuration for the user interface
|
|
||||||
type UI struct {
|
type UI struct {
|
||||||
PopupWidth int `yaml:"popup_width" json:"popup_width"`
|
PopupWidth int `yaml:"popup_width" json:"popup_width"`
|
||||||
PopupHeight int `yaml:"popup_height" json:"popup_height"`
|
PopupHeight int `yaml:"popup_height" json:"popup_height"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Destination is the path for a place that a download can be moved to
|
|
||||||
type Destination struct {
|
|
||||||
Name string `yaml:"name" json:"name"` // Name for this location
|
|
||||||
Path string `yaml:"path" json:"path"` // Path on disk
|
|
||||||
}
|
|
||||||
|
|
||||||
// Config is the top level of the user configuration
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
ConfigVersion int `yaml:"config_version" json:"config_version"`
|
ConfigVersion int `yaml:"config_version" json:"config_version"`
|
||||||
Server Server `yaml:"server" json:"server"`
|
Server Server `yaml:"server" json:"server"`
|
||||||
UI UI `yaml:"ui" json:"ui"`
|
UI UI `yaml:"ui" json:"ui"`
|
||||||
Destinations []Destination `yaml:"destinations" json:"destinations"` // no longer in use, see DownloadOptions
|
|
||||||
DownloadProfiles []DownloadProfile `yaml:"profiles" json:"profiles"`
|
DownloadProfiles []DownloadProfile `yaml:"profiles" json:"profiles"`
|
||||||
DownloadOptions []DownloadOption `yaml:"download_options" json:"download_options"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConfigService is a struct to handle configuration requests, allowing for the
|
// ConfigService is a struct to handle configuration requests, allowing for the
|
||||||
@ -64,19 +46,18 @@ type ConfigService struct {
|
|||||||
|
|
||||||
func (cs *ConfigService) LoadTestConfig() {
|
func (cs *ConfigService) LoadTestConfig() {
|
||||||
cs.LoadDefaultConfig()
|
cs.LoadDefaultConfig()
|
||||||
cs.Config.Server.DownloadPath = "/tmp"
|
cs.Config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "sleep", Args: []string{"5"}}}
|
||||||
cs.Config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "/bin/sleep", Args: []string{"5"}}}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cs *ConfigService) LoadDefaultConfig() {
|
func (cs *ConfigService) LoadDefaultConfig() {
|
||||||
defaultConfig := Config{}
|
defaultConfig := Config{}
|
||||||
stdProfile := DownloadProfile{Name: "standard video", Command: "yt-dlp", Args: []string{
|
stdProfile := DownloadProfile{Name: "standard video", Command: "youtube-dl", Args: []string{
|
||||||
"--newline",
|
"--newline",
|
||||||
"--write-info-json",
|
"--write-info-json",
|
||||||
"-f",
|
"-f",
|
||||||
"bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
|
"bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
|
||||||
}}
|
}}
|
||||||
mp3Profile := DownloadProfile{Name: "standard mp3", Command: "yt-dlp", Args: []string{
|
mp3Profile := DownloadProfile{Name: "standard mp3", Command: "youtube-dl", Args: []string{
|
||||||
"--newline",
|
"--newline",
|
||||||
"--write-info-json",
|
"--write-info-json",
|
||||||
"--extract-audio",
|
"--extract-audio",
|
||||||
@ -88,23 +69,20 @@ func (cs *ConfigService) LoadDefaultConfig() {
|
|||||||
|
|
||||||
defaultConfig.Server.Port = 6123
|
defaultConfig.Server.Port = 6123
|
||||||
defaultConfig.Server.Address = "http://localhost:6123"
|
defaultConfig.Server.Address = "http://localhost:6123"
|
||||||
defaultConfig.Server.DownloadPath = "/downloads"
|
defaultConfig.Server.DownloadPath = "./"
|
||||||
|
|
||||||
defaultConfig.UI.PopupWidth = 500
|
defaultConfig.UI.PopupWidth = 500
|
||||||
defaultConfig.UI.PopupHeight = 500
|
defaultConfig.UI.PopupHeight = 500
|
||||||
|
|
||||||
defaultConfig.Server.MaximumActiveDownloads = 2
|
defaultConfig.Server.MaximumActiveDownloads = 2
|
||||||
|
|
||||||
defaultConfig.Destinations = nil
|
defaultConfig.ConfigVersion = 2
|
||||||
defaultConfig.DownloadOptions = make([]DownloadOption, 0)
|
|
||||||
|
|
||||||
defaultConfig.ConfigVersion = 4
|
|
||||||
|
|
||||||
cs.Config = &defaultConfig
|
cs.Config = &defaultConfig
|
||||||
|
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProfileCalled returns the corresponding DownloadProfile, or nil if it does not exist
|
|
||||||
func (c *Config) ProfileCalled(name string) *DownloadProfile {
|
func (c *Config) ProfileCalled(name string) *DownloadProfile {
|
||||||
for _, p := range c.DownloadProfiles {
|
for _, p := range c.DownloadProfiles {
|
||||||
if p.Name == name {
|
if p.Name == name {
|
||||||
@ -114,16 +92,6 @@ func (c *Config) ProfileCalled(name string) *DownloadProfile {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DownloadOptionCalled returns the corresponding DownloadOption, or nil if it does not exist
|
|
||||||
func (c *Config) DownloadOptionCalled(name string) *DownloadOption {
|
|
||||||
for _, o := range c.DownloadOptions {
|
|
||||||
if o.Name == name {
|
|
||||||
return &o
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Config) UpdateFromJSON(j []byte) error {
|
func (c *Config) UpdateFromJSON(j []byte) error {
|
||||||
newConfig := Config{}
|
newConfig := Config{}
|
||||||
err := json.Unmarshal(j, &newConfig)
|
err := json.Unmarshal(j, &newConfig)
|
||||||
@ -189,10 +157,9 @@ func (c *Config) UpdateFromJSON(j []byte) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// check the command exists
|
// check the command exists
|
||||||
|
_, err := exec.LookPath(newConfig.DownloadProfiles[i].Command)
|
||||||
_, err := AbsPathToExecutable(newConfig.DownloadProfiles[i].Command)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("problem with command '%s': %s", newConfig.DownloadProfiles[i].Command, err)
|
return fmt.Errorf("Could not find %s on the path", newConfig.DownloadProfiles[i].Command)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -202,24 +169,12 @@ func (c *Config) UpdateFromJSON(j []byte) error {
|
|||||||
|
|
||||||
// DetermineConfigDir determines where the config is (or should be) stored.
|
// DetermineConfigDir determines where the config is (or should be) stored.
|
||||||
func (cs *ConfigService) DetermineConfigDir() {
|
func (cs *ConfigService) DetermineConfigDir() {
|
||||||
// check binary path first, for a file called gropple.yml
|
// check current directory first, for a file called gropple.yml
|
||||||
binaryPath := os.Args[0]
|
_, err := os.Stat("gropple.yml")
|
||||||
binaryDir := filepath.Dir(binaryPath)
|
|
||||||
potentialConfigPath := filepath.Join(binaryDir, "gropple.yml")
|
|
||||||
|
|
||||||
_, err := os.Stat(potentialConfigPath)
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
// exists in binary directory, use that
|
// exists in current directory, use that.
|
||||||
// fully qualify, just for clarity in the log
|
cs.ConfigPath = "gropple.yml"
|
||||||
config, err := filepath.Abs(potentialConfigPath)
|
return
|
||||||
if err == nil {
|
|
||||||
log.Printf("found portable config in %s", config)
|
|
||||||
cs.ConfigPath = config
|
|
||||||
return
|
|
||||||
} else {
|
|
||||||
log.Printf("got error when trying to convert config to absolute path: %s", err)
|
|
||||||
log.Print("falling back to using UserConfigDir")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// otherwise fall back to using the UserConfigDir
|
// otherwise fall back to using the UserConfigDir
|
||||||
@ -266,14 +221,12 @@ func (cs *ConfigService) LoadConfig() error {
|
|||||||
path := cs.ConfigPath
|
path := cs.ConfigPath
|
||||||
b, err := os.ReadFile(path)
|
b, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not read config '%s': %v", path, err)
|
return fmt.Errorf("Could not read config '%s': %v", path, err)
|
||||||
}
|
}
|
||||||
c := Config{}
|
c := Config{}
|
||||||
cs.Config = &c
|
|
||||||
|
|
||||||
err = yaml.Unmarshal(b, &c)
|
err = yaml.Unmarshal(b, &c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not parse YAML config '%s': %v", path, err)
|
return fmt.Errorf("Could not parse YAML config '%s': %v", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// do migrations
|
// do migrations
|
||||||
@ -283,28 +236,6 @@ func (cs *ConfigService) LoadConfig() error {
|
|||||||
c.ConfigVersion = 2
|
c.ConfigVersion = 2
|
||||||
configMigrated = true
|
configMigrated = true
|
||||||
log.Print("migrated config from version 1 => 2")
|
log.Print("migrated config from version 1 => 2")
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if c.ConfigVersion == 2 {
|
|
||||||
c.Destinations = make([]Destination, 0)
|
|
||||||
c.ConfigVersion = 3
|
|
||||||
configMigrated = true
|
|
||||||
log.Print("migrated config from version 2 => 3")
|
|
||||||
}
|
|
||||||
|
|
||||||
if c.ConfigVersion == 3 {
|
|
||||||
c.ConfigVersion = 4
|
|
||||||
for i := range c.Destinations {
|
|
||||||
newDownloadOption := DownloadOption{
|
|
||||||
Name: c.Destinations[i].Name,
|
|
||||||
Args: []string{"-o", fmt.Sprintf("%s/%%(title)s [%%(id)s].%%(ext)s", c.Destinations[i].Path)},
|
|
||||||
}
|
|
||||||
c.DownloadOptions = append(c.DownloadOptions, newDownloadOption)
|
|
||||||
}
|
|
||||||
c.Destinations = nil
|
|
||||||
configMigrated = true
|
|
||||||
log.Print("migrated config from version 3 => 4")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if configMigrated {
|
if configMigrated {
|
||||||
@ -312,6 +243,8 @@ func (cs *ConfigService) LoadConfig() error {
|
|||||||
cs.WriteConfig()
|
cs.WriteConfig()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cs.Config = &c
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -319,8 +252,7 @@ func (cs *ConfigService) LoadConfig() error {
|
|||||||
func (cs *ConfigService) WriteConfig() {
|
func (cs *ConfigService) WriteConfig() {
|
||||||
s, err := yaml.Marshal(cs.Config)
|
s, err := yaml.Marshal(cs.Config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("error writing config: %s", err)
|
panic(err)
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
path := cs.ConfigPath
|
path := cs.ConfigPath
|
||||||
@ -333,34 +265,6 @@ func (cs *ConfigService) WriteConfig() {
|
|||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
|
||||||
_, err = file.Write(s)
|
file.Write(s)
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("could not write config file %s: %s", path, err)
|
|
||||||
}
|
|
||||||
file.Close()
|
file.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
// AbsPathToExecutable takes a command name, which may or may not be path-qualified,
|
|
||||||
// and returns the fully qualified path to it, or an error if could not be found, or
|
|
||||||
// if it does not appear to be a file.
|
|
||||||
func AbsPathToExecutable(cmd string) (string, error) {
|
|
||||||
|
|
||||||
pathCmd, err := exec.LookPath(cmd)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("could not LookPath '%s': %w", cmd, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
execAbsolutePath, err := filepath.Abs(pathCmd)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("could not get absolute path to '%s': %w", cmd, err)
|
|
||||||
}
|
|
||||||
fi, err := os.Stat(execAbsolutePath)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("could not get stat '%s': %w", cmd, err)
|
|
||||||
}
|
|
||||||
if !fi.Mode().IsRegular() {
|
|
||||||
return "", fmt.Errorf("'%s' is not a regular file: %w", cmd, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return execAbsolutePath, nil
|
|
||||||
}
|
|
||||||
|
|||||||
@ -1,220 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMigrationV1toV4(t *testing.T) {
|
|
||||||
v1Config := `config_version: 1
|
|
||||||
server:
|
|
||||||
port: 6123
|
|
||||||
address: http://localhost:6123
|
|
||||||
download_path: ./
|
|
||||||
ui:
|
|
||||||
popup_width: 500
|
|
||||||
popup_height: 500
|
|
||||||
profiles:
|
|
||||||
- name: standard video
|
|
||||||
command: youtube-dl
|
|
||||||
args:
|
|
||||||
- --newline
|
|
||||||
- --write-info-json
|
|
||||||
- -f
|
|
||||||
- bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best
|
|
||||||
- name: standard mp3
|
|
||||||
command: youtube-dl
|
|
||||||
args:
|
|
||||||
- --newline
|
|
||||||
- --write-info-json
|
|
||||||
- --extract-audio
|
|
||||||
- --audio-format
|
|
||||||
- mp3
|
|
||||||
`
|
|
||||||
cs := configServiceFromString(v1Config)
|
|
||||||
err := cs.LoadConfig()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("got error when loading config: %s", err)
|
|
||||||
}
|
|
||||||
if cs.Config.ConfigVersion != 4 {
|
|
||||||
t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion)
|
|
||||||
}
|
|
||||||
if cs.Config.Server.MaximumActiveDownloads != 2 {
|
|
||||||
t.Error("did not add MaximumActiveDownloads")
|
|
||||||
}
|
|
||||||
if len(cs.Config.Destinations) != 0 {
|
|
||||||
t.Error("incorrect number of destinations added")
|
|
||||||
}
|
|
||||||
os.Remove(cs.ConfigPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMigrateV3toV4(t *testing.T) {
|
|
||||||
v3Config := `config_version: 3
|
|
||||||
server:
|
|
||||||
port: 6123
|
|
||||||
address: http://localhost:6123
|
|
||||||
download_path: /tmp/Downloads
|
|
||||||
maximum_active_downloads_per_domain: 2
|
|
||||||
ui:
|
|
||||||
popup_width: 900
|
|
||||||
popup_height: 900
|
|
||||||
destinations:
|
|
||||||
- name: cool destination
|
|
||||||
path: /tmp/coolness
|
|
||||||
profiles:
|
|
||||||
- name: standard video
|
|
||||||
command: yt-dlp
|
|
||||||
args:
|
|
||||||
- --newline
|
|
||||||
- --write-info-json
|
|
||||||
- -f
|
|
||||||
- bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best
|
|
||||||
- name: standard mp3
|
|
||||||
command: yt-dlp
|
|
||||||
args:
|
|
||||||
- --newline
|
|
||||||
- --write-info-json
|
|
||||||
- --extract-audio
|
|
||||||
- --audio-format
|
|
||||||
- mp3`
|
|
||||||
cs := configServiceFromString(v3Config)
|
|
||||||
err := cs.LoadConfig()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("got error when loading config: %s", err)
|
|
||||||
}
|
|
||||||
if cs.Config.ConfigVersion != 4 {
|
|
||||||
t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion)
|
|
||||||
}
|
|
||||||
if cs.Config.Server.MaximumActiveDownloads != 2 {
|
|
||||||
t.Error("did not add MaximumActiveDownloads")
|
|
||||||
}
|
|
||||||
if len(cs.Config.Destinations) != 0 {
|
|
||||||
t.Error("incorrect number of destinations from migrated file")
|
|
||||||
}
|
|
||||||
if assert.Len(t, cs.Config.DownloadOptions, 1) {
|
|
||||||
if assert.Len(t, cs.Config.DownloadOptions[0].Args, 2) {
|
|
||||||
assert.Equal(t, "-o", cs.Config.DownloadOptions[0].Args[0])
|
|
||||||
assert.Equal(t, "/tmp/coolness/%(title)s [%(id)s].%(ext)s", cs.Config.DownloadOptions[0].Args[1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
os.Remove(cs.ConfigPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMigrateV3toV4CrashBug(t *testing.T) {
|
|
||||||
v3Config := `config_version: 3
|
|
||||||
server:
|
|
||||||
port: 6123
|
|
||||||
address: https://superaddress.here.com
|
|
||||||
download_path: /home/path/gropple
|
|
||||||
maximum_active_downloads_per_domain: 2
|
|
||||||
ui:
|
|
||||||
popup_width: 500
|
|
||||||
popup_height: 500
|
|
||||||
destinations:
|
|
||||||
- name: somegifs
|
|
||||||
path: /home/path/somegifs
|
|
||||||
- name: otherstuff
|
|
||||||
path: /home/path/otherstuff
|
|
||||||
profiles:
|
|
||||||
- name: standard video
|
|
||||||
command: yt-dlp
|
|
||||||
args:
|
|
||||||
- --newline
|
|
||||||
- --write-info-json
|
|
||||||
- -f
|
|
||||||
- bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best
|
|
||||||
- --verbose
|
|
||||||
- --embed-metadata
|
|
||||||
- --embed-subs
|
|
||||||
- --embed-thumbnail
|
|
||||||
- name: standard mp3
|
|
||||||
command: yt-dlp
|
|
||||||
args:
|
|
||||||
- --extract-audio
|
|
||||||
- --audio-format
|
|
||||||
- mp3
|
|
||||||
- --prefer-ffmpeg
|
|
||||||
`
|
|
||||||
cs := configServiceFromString(v3Config)
|
|
||||||
err := cs.LoadConfig()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("got error when loading config: %s", err)
|
|
||||||
}
|
|
||||||
if cs.Config.ConfigVersion != 4 {
|
|
||||||
t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion)
|
|
||||||
}
|
|
||||||
if cs.Config.Server.MaximumActiveDownloads != 2 {
|
|
||||||
t.Error("did not add MaximumActiveDownloads")
|
|
||||||
}
|
|
||||||
if len(cs.Config.Destinations) != 0 {
|
|
||||||
t.Error("incorrect number of destinations from migrated file")
|
|
||||||
}
|
|
||||||
if assert.Len(t, cs.Config.DownloadOptions, 2) {
|
|
||||||
if assert.Len(t, cs.Config.DownloadOptions[0].Args, 2) {
|
|
||||||
assert.Equal(t, "-o", cs.Config.DownloadOptions[0].Args[0])
|
|
||||||
assert.Equal(t, "/home/path/somegifs/%(title)s [%(id)s].%(ext)s", cs.Config.DownloadOptions[0].Args[1])
|
|
||||||
assert.Equal(t, "-o", cs.Config.DownloadOptions[1].Args[0])
|
|
||||||
assert.Equal(t, "/home/path/otherstuff/%(title)s [%(id)s].%(ext)s", cs.Config.DownloadOptions[1].Args[1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
os.Remove(cs.ConfigPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
func configServiceFromString(configString string) *ConfigService {
|
|
||||||
tmpFile, _ := os.CreateTemp("", "gropple_test_*.yml")
|
|
||||||
_, err1 := tmpFile.Write([]byte(configString))
|
|
||||||
err2 := tmpFile.Close()
|
|
||||||
|
|
||||||
if errors.Join(err1, err2) != nil {
|
|
||||||
panic("got unexpected error")
|
|
||||||
}
|
|
||||||
|
|
||||||
cs := ConfigService{
|
|
||||||
Config: &Config{},
|
|
||||||
ConfigPath: tmpFile.Name(),
|
|
||||||
}
|
|
||||||
return &cs
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLookForExecutable(t *testing.T) {
|
|
||||||
cmdPath, err := exec.LookPath("sleep")
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("cannot run this test without knowing about sleep: %s", err)
|
|
||||||
t.FailNow()
|
|
||||||
}
|
|
||||||
cmdDir := filepath.Dir(cmdPath)
|
|
||||||
|
|
||||||
cmd := "sleep"
|
|
||||||
path, err := AbsPathToExecutable(cmd)
|
|
||||||
if assert.NoError(t, err) {
|
|
||||||
assert.Equal(t, cmdPath, path)
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd = cmdPath
|
|
||||||
path, err = AbsPathToExecutable(cmd)
|
|
||||||
if assert.NoError(t, err) {
|
|
||||||
assert.Equal(t, cmdPath, path)
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd = "../../../../../../../../.." + cmdPath
|
|
||||||
path, err = AbsPathToExecutable(cmd)
|
|
||||||
if assert.NoError(t, err) {
|
|
||||||
assert.Equal(t, cmdPath, path)
|
|
||||||
}
|
|
||||||
cmd = "./sleep"
|
|
||||||
_, err = AbsPathToExecutable(cmd)
|
|
||||||
assert.Error(t, err)
|
|
||||||
|
|
||||||
os.Chdir(cmdDir) //nolint
|
|
||||||
cmd = "./sleep"
|
|
||||||
path, err = AbsPathToExecutable(cmd)
|
|
||||||
if assert.NoError(t, err) {
|
|
||||||
assert.Equal(t, cmdPath, path)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,11 +0,0 @@
|
|||||||
version: "3.9"
|
|
||||||
|
|
||||||
services:
|
|
||||||
gropple:
|
|
||||||
image: tardisx/gropple:v1.1.4
|
|
||||||
volumes:
|
|
||||||
- /tmp/gropple-config-dir/:/config
|
|
||||||
- /tmp/downloads/:/downloads/
|
|
||||||
restart: always
|
|
||||||
ports:
|
|
||||||
- "6123:6123"
|
|
||||||
@ -1,14 +1,12 @@
|
|||||||
package download
|
package download
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@ -25,203 +23,118 @@ type Download struct {
|
|||||||
PopupUrl string `json:"popup_url"`
|
PopupUrl string `json:"popup_url"`
|
||||||
Process *os.Process `json:"-"`
|
Process *os.Process `json:"-"`
|
||||||
ExitCode int `json:"exit_code"`
|
ExitCode int `json:"exit_code"`
|
||||||
State State `json:"state"`
|
State string `json:"state"`
|
||||||
DownloadProfile config.DownloadProfile `json:"download_profile"`
|
DownloadProfile config.DownloadProfile `json:"download_profile"`
|
||||||
DownloadOption *config.DownloadOption `json:"download_option"`
|
|
||||||
Finished bool `json:"finished"`
|
Finished bool `json:"finished"`
|
||||||
FinishedTS time.Time `json:"finished_ts"`
|
FinishedTS time.Time `json:"finished_ts"`
|
||||||
Files []string `json:"files"`
|
Files []string `json:"files"`
|
||||||
PlaylistCurrent int `json:"playlist_current"`
|
|
||||||
PlaylistTotal int `json:"playlist_total"`
|
|
||||||
Eta string `json:"eta"`
|
Eta string `json:"eta"`
|
||||||
Percent float32 `json:"percent"`
|
Percent float32 `json:"percent"`
|
||||||
Log []string `json:"log"`
|
Log []string `json:"log"`
|
||||||
Config *config.Config
|
Config *config.Config
|
||||||
Lock sync.Mutex
|
mutex sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
// The Manager holds and is responsible for all Download objects.
|
type Downloads []*Download
|
||||||
type Manager struct {
|
|
||||||
Downloads []*Download
|
|
||||||
MaxPerDomain int
|
|
||||||
Lock sync.Mutex
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Manager) String() string {
|
|
||||||
m.Lock.Lock()
|
|
||||||
defer m.Lock.Unlock()
|
|
||||||
out := fmt.Sprintf("Max per domain: %d, downloads: %d\n", m.MaxPerDomain, len(m.Downloads))
|
|
||||||
|
|
||||||
for _, dl := range m.Downloads {
|
|
||||||
out = out + fmt.Sprintf("%3d: (%10s) %30s\n", dl.Id, dl.State, dl.Url)
|
|
||||||
}
|
|
||||||
|
|
||||||
return out
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
type State string
|
|
||||||
|
|
||||||
const (
|
|
||||||
STATE_PREPARING State = "Preparing to start"
|
|
||||||
STATE_CHOOSE_PROFILE State = "Choose Profile"
|
|
||||||
STATE_QUEUED State = "Queued"
|
|
||||||
STATE_DOWNLOADING State = "Downloading"
|
|
||||||
STATE_DOWNLOADING_METADATA State = "Downloading metadata"
|
|
||||||
STATE_FAILED State = "Failed"
|
|
||||||
STATE_COMPLETE State = "Complete"
|
|
||||||
STATE_MOVED State = "Moved"
|
|
||||||
)
|
|
||||||
|
|
||||||
var CanStopDownload = false
|
var CanStopDownload = false
|
||||||
|
|
||||||
var downloadId int32 = 0
|
var downloadId int32 = 0
|
||||||
|
|
||||||
func (m *Manager) ManageQueue() {
|
// StartQueued starts any downloads that have been queued, we would not exceed
|
||||||
for {
|
|
||||||
m.Lock.Lock()
|
|
||||||
|
|
||||||
m.startQueued(m.MaxPerDomain)
|
|
||||||
m.cleanup()
|
|
||||||
m.Lock.Unlock()
|
|
||||||
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Manager) DownloadsAsJSON() ([]byte, error) {
|
|
||||||
|
|
||||||
m.Lock.Lock()
|
|
||||||
defer m.Lock.Unlock()
|
|
||||||
for _, dl := range m.Downloads {
|
|
||||||
dl.Lock.Lock()
|
|
||||||
defer dl.Lock.Unlock()
|
|
||||||
}
|
|
||||||
b, err := json.Marshal(m.Downloads)
|
|
||||||
return b, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// startQueued starts any downloads that have been queued, we would not exceed
|
|
||||||
// maxRunning. If maxRunning is 0, there is no limit.
|
// maxRunning. If maxRunning is 0, there is no limit.
|
||||||
func (m *Manager) startQueued(maxRunning int) {
|
func (dls Downloads) StartQueued(maxRunning int) {
|
||||||
|
|
||||||
active := make(map[string]int)
|
active := make(map[string]int)
|
||||||
|
|
||||||
for _, dl := range m.Downloads {
|
for _, dl := range dls {
|
||||||
dl.Lock.Lock()
|
|
||||||
|
|
||||||
if dl.State == STATE_DOWNLOADING || dl.State == STATE_PREPARING {
|
dl.mutex.Lock()
|
||||||
|
|
||||||
|
if dl.State == "downloading" {
|
||||||
active[dl.domain()]++
|
active[dl.domain()]++
|
||||||
}
|
}
|
||||||
dl.Lock.Unlock()
|
dl.mutex.Unlock()
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, dl := range m.Downloads {
|
for _, dl := range dls {
|
||||||
|
|
||||||
dl.Lock.Lock()
|
dl.mutex.Lock()
|
||||||
|
|
||||||
if dl.State == STATE_QUEUED && (maxRunning == 0 || active[dl.domain()] < maxRunning) {
|
if dl.State == "queued" && (maxRunning == 0 || active[dl.domain()] < maxRunning) {
|
||||||
dl.State = STATE_PREPARING
|
dl.State = "downloading"
|
||||||
active[dl.domain()]++
|
active[dl.domain()]++
|
||||||
log.Printf("Starting download for id:%d (%s)", dl.Id, dl.Url)
|
log.Printf("Starting download for id:%d (%s)", dl.Id, dl.Url)
|
||||||
|
dl.mutex.Unlock()
|
||||||
dl.Lock.Unlock()
|
go func() { dl.Begin() }()
|
||||||
|
|
||||||
go func(sdl *Download) {
|
|
||||||
sdl.Begin()
|
|
||||||
}(dl)
|
|
||||||
} else {
|
} else {
|
||||||
dl.Lock.Unlock()
|
dl.mutex.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// cleanup removes old downloads from the list. Hardcoded to remove them one hour
|
// Cleanup removes old downloads from the list. Hardcoded to remove them one hour
|
||||||
// completion. Expects the Manager to be locked.
|
// completion.
|
||||||
func (m *Manager) cleanup() {
|
func (dls Downloads) Cleanup() Downloads {
|
||||||
newDLs := []*Download{}
|
newDLs := Downloads{}
|
||||||
for _, dl := range m.Downloads {
|
for _, dl := range dls {
|
||||||
dl.Lock.Lock()
|
|
||||||
|
dl.mutex.Lock()
|
||||||
|
|
||||||
if dl.Finished && time.Since(dl.FinishedTS) > time.Duration(time.Hour) {
|
if dl.Finished && time.Since(dl.FinishedTS) > time.Duration(time.Hour) {
|
||||||
// do nothing
|
// do nothing
|
||||||
} else {
|
} else {
|
||||||
newDLs = append(newDLs, dl)
|
newDLs = append(newDLs, dl)
|
||||||
}
|
}
|
||||||
dl.Lock.Unlock()
|
dl.mutex.Unlock()
|
||||||
|
|
||||||
}
|
}
|
||||||
m.Downloads = newDLs
|
return newDLs
|
||||||
}
|
|
||||||
|
|
||||||
// GetDlById returns one of the downloads in our current list.
|
|
||||||
func (m *Manager) GetDlById(id int) (*Download, error) {
|
|
||||||
m.Lock.Lock()
|
|
||||||
defer m.Lock.Unlock()
|
|
||||||
for _, dl := range m.Downloads {
|
|
||||||
if dl.Id == id {
|
|
||||||
return dl, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("no download with id %d", id)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Queue queues a download
|
// Queue queues a download
|
||||||
func (m *Manager) Queue(dl *Download) {
|
func (dl *Download) Queue() {
|
||||||
dl.Lock.Lock()
|
|
||||||
defer dl.Lock.Unlock()
|
dl.mutex.Lock()
|
||||||
dl.State = STATE_QUEUED
|
defer dl.mutex.Unlock()
|
||||||
|
|
||||||
|
dl.State = "queued"
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewDownload(url string, conf *config.Config) *Download {
|
func NewDownload(conf *config.Config, url string) *Download {
|
||||||
atomic.AddInt32(&downloadId, 1)
|
atomic.AddInt32(&downloadId, 1)
|
||||||
dl := Download{
|
dl := Download{
|
||||||
|
Config: conf,
|
||||||
|
|
||||||
Id: int(downloadId),
|
Id: int(downloadId),
|
||||||
Url: url,
|
Url: url,
|
||||||
PopupUrl: fmt.Sprintf("/fetch/%d", int(downloadId)),
|
PopupUrl: fmt.Sprintf("/fetch/%d", int(downloadId)),
|
||||||
State: STATE_CHOOSE_PROFILE,
|
State: "choose profile",
|
||||||
Files: []string{},
|
Finished: false,
|
||||||
Log: []string{},
|
Eta: "?",
|
||||||
Config: conf,
|
Percent: 0.0,
|
||||||
Lock: sync.Mutex{},
|
Log: make([]string, 0, 1000),
|
||||||
}
|
}
|
||||||
return &dl
|
return &dl
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Manager) AddDownload(dl *Download) {
|
|
||||||
m.Lock.Lock()
|
|
||||||
defer m.Lock.Unlock()
|
|
||||||
m.Downloads = append(m.Downloads, dl)
|
|
||||||
}
|
|
||||||
|
|
||||||
// func (dl *Download) AppendLog(text string) {
|
|
||||||
// dl.Lock.Lock()
|
|
||||||
// defer dl.Lock.Unlock()
|
|
||||||
// dl.Log = append(dl.Log, text)
|
|
||||||
// }
|
|
||||||
|
|
||||||
// Stop the download.
|
|
||||||
func (dl *Download) Stop() {
|
func (dl *Download) Stop() {
|
||||||
if !CanStopDownload {
|
if !CanStopDownload {
|
||||||
log.Print("attempted to stop download on a platform that it is not currently supported on - please report this as a bug")
|
log.Print("attempted to stop download on a platform that it is not currently supported on - please report this as a bug")
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("stopping the download")
|
log.Printf("stopping the download")
|
||||||
dl.Lock.Lock()
|
dl.mutex.Lock()
|
||||||
defer dl.Lock.Unlock()
|
|
||||||
dl.Log = append(dl.Log, "aborted by user")
|
dl.Log = append(dl.Log, "aborted by user")
|
||||||
err := dl.Process.Kill()
|
defer dl.mutex.Unlock()
|
||||||
if err != nil {
|
dl.Process.Kill()
|
||||||
log.Printf("could not send kill to process: %s", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// domain returns a domain for this Download. Download should be locked.
|
|
||||||
func (dl *Download) domain() string {
|
func (dl *Download) domain() string {
|
||||||
|
|
||||||
|
// note that we expect to already have the mutex locked by the caller
|
||||||
url, err := url.Parse(dl.Url)
|
url, err := url.Parse(dl.Url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Unknown domain for url: %s", dl.Url)
|
log.Printf("Unknown domain for url: %s", dl.Url)
|
||||||
@ -235,109 +148,55 @@ func (dl *Download) domain() string {
|
|||||||
// Begin starts a download, by starting the command specified in the DownloadProfile.
|
// Begin starts a download, by starting the command specified in the DownloadProfile.
|
||||||
// It blocks until the download is complete.
|
// It blocks until the download is complete.
|
||||||
func (dl *Download) Begin() {
|
func (dl *Download) Begin() {
|
||||||
dl.Lock.Lock()
|
|
||||||
u, err := url.Parse(dl.Url)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Bad url '%s': %s", dl.Url, err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
// grab the host and path for substitutions
|
dl.mutex.Lock()
|
||||||
host := u.Host
|
|
||||||
path := u.Path
|
|
||||||
|
|
||||||
// strip the leading /
|
dl.State = "downloading"
|
||||||
if strings.Index(path, "/") == 0 {
|
|
||||||
path = path[1:]
|
|
||||||
}
|
|
||||||
|
|
||||||
// escape them in a way that should mean we can use them as a filepath
|
|
||||||
host = strings.ReplaceAll(host, string(filepath.Separator), "_")
|
|
||||||
host = strings.ReplaceAll(host, string(filepath.ListSeparator), "_")
|
|
||||||
path = strings.ReplaceAll(path, string(filepath.Separator), "_")
|
|
||||||
path = strings.ReplaceAll(path, string(filepath.ListSeparator), "_")
|
|
||||||
|
|
||||||
dl.State = STATE_DOWNLOADING
|
|
||||||
cmdSlice := []string{}
|
cmdSlice := []string{}
|
||||||
|
cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...)
|
||||||
for i := range dl.DownloadProfile.Args {
|
|
||||||
arg := dl.DownloadProfile.Args[i]
|
|
||||||
arg = strings.ReplaceAll(arg, "%GROPPLE_HOST%", host)
|
|
||||||
arg = strings.ReplaceAll(arg, "%GROPPLE_PATH%", path)
|
|
||||||
cmdSlice = append(cmdSlice, arg)
|
|
||||||
}
|
|
||||||
|
|
||||||
// add the option, if any
|
|
||||||
if dl.DownloadOption != nil {
|
|
||||||
for i := range dl.DownloadOption.Args {
|
|
||||||
arg := dl.DownloadOption.Args[i]
|
|
||||||
arg = strings.ReplaceAll(arg, "%GROPPLE_HOST%", host)
|
|
||||||
arg = strings.ReplaceAll(arg, "%GROPPLE_PATH%", path)
|
|
||||||
cmdSlice = append(cmdSlice, arg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// only add the url if it's not empty or an example URL. This helps us with testing
|
// only add the url if it's not empty or an example URL. This helps us with testing
|
||||||
if dl.Url != "" && !strings.Contains(dl.domain(), "example.org") {
|
if !(dl.Url == "" || strings.Contains(dl.domain(), "example.org")) {
|
||||||
cmdSlice = append(cmdSlice, dl.Url)
|
cmdSlice = append(cmdSlice, dl.Url)
|
||||||
}
|
}
|
||||||
|
|
||||||
cmdPath, err := config.AbsPathToExecutable(dl.DownloadProfile.Command)
|
cmd := exec.Command(dl.DownloadProfile.Command, cmdSlice...)
|
||||||
if err != nil {
|
|
||||||
dl.State = STATE_FAILED
|
|
||||||
dl.Finished = true
|
|
||||||
dl.FinishedTS = time.Now()
|
|
||||||
dl.Log = append(dl.Log, fmt.Sprintf("error finding executable for downloader: %s", err.Error()))
|
|
||||||
dl.Lock.Unlock()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
dl.Log = append(dl.Log, fmt.Sprintf("executing: %s (%s) with args: %s", dl.DownloadProfile.Command, cmdPath, strings.Join(cmdSlice, " ")))
|
|
||||||
|
|
||||||
cmd := exec.Command(cmdPath, cmdSlice...)
|
|
||||||
cmd.Dir = dl.Config.Server.DownloadPath
|
cmd.Dir = dl.Config.Server.DownloadPath
|
||||||
log.Printf("Executing command executable: %s) in %s", cmdPath, dl.Config.Server.DownloadPath)
|
|
||||||
|
|
||||||
stdout, err := cmd.StdoutPipe()
|
stdout, err := cmd.StdoutPipe()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
dl.State = STATE_FAILED
|
dl.State = "failed"
|
||||||
dl.Finished = true
|
dl.Finished = true
|
||||||
dl.FinishedTS = time.Now()
|
dl.FinishedTS = time.Now()
|
||||||
dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
|
dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
|
||||||
dl.Lock.Unlock()
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
stderr, err := cmd.StderrPipe()
|
stderr, err := cmd.StderrPipe()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
dl.State = STATE_FAILED
|
dl.State = "failed"
|
||||||
dl.Finished = true
|
dl.Finished = true
|
||||||
dl.FinishedTS = time.Now()
|
dl.FinishedTS = time.Now()
|
||||||
dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
|
dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
|
||||||
dl.Lock.Unlock()
|
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Printf("Executing command: %v", cmd)
|
||||||
err = cmd.Start()
|
err = cmd.Start()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Executing command failed: %s", err.Error())
|
dl.State = "failed"
|
||||||
|
|
||||||
dl.State = STATE_FAILED
|
|
||||||
dl.Finished = true
|
dl.Finished = true
|
||||||
dl.FinishedTS = time.Now()
|
dl.FinishedTS = time.Now()
|
||||||
dl.Log = append(dl.Log, fmt.Sprintf("error starting command '%s': %v", dl.DownloadProfile.Command, err))
|
dl.Log = append(dl.Log, fmt.Sprintf("error starting command '%s': %v", dl.DownloadProfile.Command, err))
|
||||||
dl.Lock.Unlock()
|
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
dl.Process = cmd.Process
|
dl.Process = cmd.Process
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
dl.mutex.Unlock()
|
||||||
|
|
||||||
wg.Add(2)
|
wg.Add(2)
|
||||||
|
|
||||||
dl.Lock.Unlock()
|
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
dl.updateDownload(stdout)
|
dl.updateDownload(stdout)
|
||||||
@ -349,38 +208,24 @@ func (dl *Download) Begin() {
|
|||||||
}()
|
}()
|
||||||
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
cmd.Wait()
|
||||||
|
|
||||||
err = cmd.Wait()
|
dl.mutex.Lock()
|
||||||
dl.Lock.Lock()
|
log.Printf("Process finished for id: %d (%v)", dl.Id, cmd)
|
||||||
|
|
||||||
if err != nil {
|
dl.State = "complete"
|
||||||
log.Printf("process failed for id: %d: %s", dl.Id, err)
|
dl.Finished = true
|
||||||
|
dl.FinishedTS = time.Now()
|
||||||
|
dl.ExitCode = cmd.ProcessState.ExitCode()
|
||||||
|
|
||||||
dl.State = STATE_FAILED
|
if dl.ExitCode != 0 {
|
||||||
dl.Finished = true
|
dl.State = "failed"
|
||||||
dl.FinishedTS = time.Now()
|
|
||||||
dl.ExitCode = cmd.ProcessState.ExitCode()
|
|
||||||
|
|
||||||
} else {
|
|
||||||
|
|
||||||
log.Printf("process finished for id: %d (%v)", dl.Id, cmd)
|
|
||||||
|
|
||||||
dl.State = STATE_COMPLETE
|
|
||||||
dl.Finished = true
|
|
||||||
dl.FinishedTS = time.Now()
|
|
||||||
dl.ExitCode = cmd.ProcessState.ExitCode()
|
|
||||||
|
|
||||||
if dl.ExitCode != 0 {
|
|
||||||
dl.State = STATE_FAILED
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
dl.Lock.Unlock()
|
dl.mutex.Unlock()
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// updateDownload updates the download based on data from the reader. Expects the
|
|
||||||
// Download to be unlocked.
|
|
||||||
func (dl *Download) updateDownload(r io.Reader) {
|
func (dl *Download) updateDownload(r io.Reader) {
|
||||||
|
|
||||||
// XXX not sure if we might get a partial line?
|
// XXX not sure if we might get a partial line?
|
||||||
buf := make([]byte, 1024)
|
buf := make([]byte, 1024)
|
||||||
for {
|
for {
|
||||||
@ -395,13 +240,15 @@ func (dl *Download) updateDownload(r io.Reader) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
dl.mutex.Lock()
|
||||||
|
|
||||||
// append the raw log
|
// append the raw log
|
||||||
dl.Lock.Lock()
|
|
||||||
dl.Log = append(dl.Log, l)
|
dl.Log = append(dl.Log, l)
|
||||||
|
|
||||||
|
dl.mutex.Unlock()
|
||||||
|
|
||||||
// look for the percent and eta and other metadata
|
// look for the percent and eta and other metadata
|
||||||
dl.updateMetadata(l)
|
dl.updateMetadata(l)
|
||||||
dl.Lock.Unlock()
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -410,16 +257,18 @@ func (dl *Download) updateDownload(r io.Reader) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// updateMetadata parses some metadata and updates the Download. Download must be locked.
|
|
||||||
func (dl *Download) updateMetadata(s string) {
|
func (dl *Download) updateMetadata(s string) {
|
||||||
|
|
||||||
|
dl.mutex.Lock()
|
||||||
|
|
||||||
|
defer dl.mutex.Unlock()
|
||||||
|
|
||||||
// [download] 49.7% of ~15.72MiB at 5.83MiB/s ETA 00:07
|
// [download] 49.7% of ~15.72MiB at 5.83MiB/s ETA 00:07
|
||||||
// [download] 99.3% of ~1.42GiB at 320.87KiB/s ETA 00:07 (frag 212/214)
|
etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d(?::\d\d)?)$`)
|
||||||
etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d(?::\d\d)?)`)
|
|
||||||
matches := etaRE.FindStringSubmatch(s)
|
matches := etaRE.FindStringSubmatch(s)
|
||||||
if len(matches) == 2 {
|
if len(matches) == 2 {
|
||||||
dl.Eta = matches[1]
|
dl.Eta = matches[1]
|
||||||
dl.State = STATE_DOWNLOADING
|
dl.State = "downloading"
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -429,6 +278,8 @@ func (dl *Download) updateMetadata(s string) {
|
|||||||
p, err := strconv.ParseFloat(matches[1], 32)
|
p, err := strconv.ParseFloat(matches[1], 32)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
dl.Percent = float32(p)
|
dl.Percent = float32(p)
|
||||||
|
} else {
|
||||||
|
panic(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -463,29 +314,4 @@ func (dl *Download) updateMetadata(s string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// [download] Downloading video 1 of 3
|
|
||||||
playlistDetails := regexp.MustCompile(`Downloading video (\d+) of (\d+)`)
|
|
||||||
matches = playlistDetails.FindStringSubmatch(s)
|
|
||||||
if len(matches) == 3 {
|
|
||||||
total, _ := strconv.ParseInt(matches[2], 10, 32)
|
|
||||||
current, _ := strconv.ParseInt(matches[1], 10, 32)
|
|
||||||
dl.PlaylistTotal = int(total)
|
|
||||||
dl.PlaylistCurrent = int(current)
|
|
||||||
}
|
|
||||||
|
|
||||||
// [Site] user: Downloading JSON metadata page 2
|
|
||||||
metadataDL := regexp.MustCompile(`Downloading JSON metadata page (\d+)`)
|
|
||||||
matches = metadataDL.FindStringSubmatch(s)
|
|
||||||
if len(matches) == 2 {
|
|
||||||
dl.State = STATE_DOWNLOADING_METADATA
|
|
||||||
}
|
|
||||||
|
|
||||||
// [FixupM3u8] Fixing MPEG-TS in MP4 container of "file [-168849776_456239489].mp4"
|
|
||||||
metadataFixup := regexp.MustCompile(`Fixing MPEG-TS in MP4 container`)
|
|
||||||
matches = metadataFixup.FindStringSubmatch(s)
|
|
||||||
if len(matches) == 1 {
|
|
||||||
dl.State = "Fixing MPEG-TS in MP4"
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,8 +0,0 @@
|
|||||||
//go:build !testdata
|
|
||||||
|
|
||||||
package download
|
|
||||||
|
|
||||||
import "github.com/tardisx/gropple/config"
|
|
||||||
|
|
||||||
func (m *Manager) AddStressTestData(c *config.ConfigService) {
|
|
||||||
}
|
|
||||||
@ -1,5 +0,0 @@
|
|||||||
package download
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
CanStopDownload = true
|
|
||||||
}
|
|
||||||
@ -1,8 +1,6 @@
|
|||||||
package download
|
package download
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -21,11 +19,11 @@ func TestUpdateMetadata(t *testing.T) {
|
|||||||
// eta's might be xx:xx:xx or xx:xx
|
// eta's might be xx:xx:xx or xx:xx
|
||||||
newD.updateMetadata("[download] 0.0% of 504.09MiB at 135.71KiB/s ETA 01:03:36")
|
newD.updateMetadata("[download] 0.0% of 504.09MiB at 135.71KiB/s ETA 01:03:36")
|
||||||
if newD.Eta != "01:03:36" {
|
if newD.Eta != "01:03:36" {
|
||||||
t.Fatalf("bad long eta in dl\n%#v", newD) //nolint
|
t.Fatalf("bad long eta in dl\n%v", newD)
|
||||||
}
|
}
|
||||||
newD.updateMetadata("[download] 0.0% of 504.09MiB at 397.98KiB/s ETA 21:38")
|
newD.updateMetadata("[download] 0.0% of 504.09MiB at 397.98KiB/s ETA 21:38")
|
||||||
if newD.Eta != "21:38" {
|
if newD.Eta != "21:38" {
|
||||||
t.Fatalf("bad short eta in dl\n%#v", newD) //nolint
|
t.Fatalf("bad short eta in dl\n%v", newD)
|
||||||
}
|
}
|
||||||
|
|
||||||
// added a new file, now we are tracking two
|
// added a new file, now we are tracking two
|
||||||
@ -41,18 +39,6 @@ func TestUpdateMetadata(t *testing.T) {
|
|||||||
t.Fatalf("%v", newD.Files)
|
t.Fatalf("%v", newD.Files)
|
||||||
}
|
}
|
||||||
|
|
||||||
// different download
|
|
||||||
newD.updateMetadata("[download] 99.3% of ~1.42GiB at 320.87KiB/s ETA 00:07 (frag 212/214)")
|
|
||||||
if newD.Eta != "00:07" {
|
|
||||||
t.Fatalf("bad short eta in dl with frag\n%v", newD) //nolint
|
|
||||||
}
|
|
||||||
|
|
||||||
// [FixupM3u8] Fixing MPEG-TS in MP4 container of "file [-168849776_456239489].mp4"
|
|
||||||
newD.updateMetadata("[FixupM3u8] Fixing MPEG-TS in MP4 container of \"file [-168849776_456239489].mp4")
|
|
||||||
if newD.State != "Fixing MPEG-TS in MP4" {
|
|
||||||
t.Fatalf("did not see fixup state - state is %s", newD.State)
|
|
||||||
}
|
|
||||||
|
|
||||||
// deletes
|
// deletes
|
||||||
// TODO. Not sure why I don't always see the "Deleting original file" messages after merge -
|
// TODO. Not sure why I don't always see the "Deleting original file" messages after merge -
|
||||||
// maybe a youtube-dl fork thing?
|
// maybe a youtube-dl fork thing?
|
||||||
@ -85,278 +71,71 @@ func TestQueue(t *testing.T) {
|
|||||||
cs.LoadTestConfig()
|
cs.LoadTestConfig()
|
||||||
conf := cs.Config
|
conf := cs.Config
|
||||||
|
|
||||||
new1 := NewDownload("http://sub.example.org/foo1", conf)
|
new1 := Download{Id: 1, Url: "http://sub.example.org/foo1", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
|
||||||
new2 := NewDownload("http://sub.example.org/foo2", conf)
|
new2 := Download{Id: 2, Url: "http://sub.example.org/foo2", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
|
||||||
new3 := NewDownload("http://sub.example.org/foo3", conf)
|
new3 := Download{Id: 3, Url: "http://sub.example.org/foo3", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
|
||||||
new4 := NewDownload("http://example.org/", conf)
|
new4 := Download{Id: 4, Url: "http://example.org/", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
|
||||||
|
|
||||||
// pretend the user chose a profile for each
|
dls := Downloads{&new1, &new2, &new3, &new4}
|
||||||
new1.DownloadProfile = *conf.ProfileCalled("test profile")
|
dls.StartQueued(1)
|
||||||
new2.DownloadProfile = *conf.ProfileCalled("test profile")
|
|
||||||
new3.DownloadProfile = *conf.ProfileCalled("test profile")
|
|
||||||
new4.DownloadProfile = *conf.ProfileCalled("test profile")
|
|
||||||
new1.State = STATE_QUEUED
|
|
||||||
new2.State = STATE_QUEUED
|
|
||||||
new3.State = STATE_QUEUED
|
|
||||||
new4.State = STATE_QUEUED
|
|
||||||
|
|
||||||
q := Manager{
|
|
||||||
Downloads: []*Download{},
|
|
||||||
MaxPerDomain: 2,
|
|
||||||
Lock: sync.Mutex{},
|
|
||||||
}
|
|
||||||
|
|
||||||
q.AddDownload(new1)
|
|
||||||
q.AddDownload(new2)
|
|
||||||
q.AddDownload(new3)
|
|
||||||
q.AddDownload(new4)
|
|
||||||
|
|
||||||
q.startQueued(1)
|
|
||||||
|
|
||||||
// two should start, one from each of the two domains
|
|
||||||
time.Sleep(time.Millisecond * 100)
|
time.Sleep(time.Millisecond * 100)
|
||||||
if q.Downloads[0].State != STATE_DOWNLOADING {
|
if dls[0].State == "queued" {
|
||||||
t.Errorf("#1 was not downloading - %s instead ", q.Downloads[0].State)
|
t.Error("#1 was not started")
|
||||||
t.Log(q.String())
|
|
||||||
}
|
}
|
||||||
if q.Downloads[1].State != STATE_QUEUED {
|
if dls[1].State != "queued" {
|
||||||
t.Errorf("#2 is not queued - %s instead", q.Downloads[1].State)
|
t.Error("#2 is not queued")
|
||||||
t.Log(q.String())
|
|
||||||
}
|
}
|
||||||
if q.Downloads[2].State != STATE_QUEUED {
|
if dls[3].State == "queued" {
|
||||||
t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
|
t.Error("#4 is not started")
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[3].State != STATE_DOWNLOADING {
|
|
||||||
t.Errorf("#4 is not downloading - %s instead", q.Downloads[3].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// this should start no more, as one is still going
|
// this should start no more, as one is still going
|
||||||
q.startQueued(1)
|
dls.StartQueued(1)
|
||||||
time.Sleep(time.Millisecond * 100)
|
time.Sleep(time.Millisecond * 100)
|
||||||
if q.Downloads[0].State != STATE_DOWNLOADING {
|
if dls[1].State != "queued" {
|
||||||
t.Errorf("#1 was not downloading - %s instead ", q.Downloads[0].State)
|
t.Error("#2 was started when it should not be")
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[1].State != STATE_QUEUED {
|
|
||||||
t.Errorf("#2 is not queued - %s instead", q.Downloads[1].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[2].State != STATE_QUEUED {
|
|
||||||
t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[3].State != STATE_DOWNLOADING {
|
|
||||||
t.Errorf("#4 is not downloading - %s instead", q.Downloads[3].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// wait until the two finish, check
|
dls.StartQueued(2)
|
||||||
time.Sleep(time.Second * 5.0)
|
time.Sleep(time.Millisecond * 100)
|
||||||
if q.Downloads[0].State != STATE_COMPLETE {
|
if dls[1].State == "queued" {
|
||||||
t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
|
t.Error("#2 was not started but it should be")
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[1].State != STATE_QUEUED {
|
|
||||||
t.Errorf("#2 is not queued - %s instead", q.Downloads[1].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[2].State != STATE_QUEUED {
|
|
||||||
t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[3].State != STATE_COMPLETE {
|
|
||||||
t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// this should start one more, as one is still going
|
dls.StartQueued(2)
|
||||||
q.startQueued(1)
|
|
||||||
time.Sleep(time.Millisecond * 100)
|
time.Sleep(time.Millisecond * 100)
|
||||||
if q.Downloads[0].State != STATE_COMPLETE {
|
if dls[3].State == "queued" {
|
||||||
t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
|
t.Error("#4 was not started but it should be")
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[1].State != STATE_DOWNLOADING {
|
|
||||||
t.Errorf("#2 is not downloading - %s instead", q.Downloads[1].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[2].State != STATE_QUEUED {
|
|
||||||
t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[3].State != STATE_COMPLETE {
|
|
||||||
t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// this should start no more, as one is still going
|
// reset them all
|
||||||
q.startQueued(1)
|
dls[0].State = "queued"
|
||||||
|
dls[1].State = "queued"
|
||||||
|
dls[2].State = "queued"
|
||||||
|
dls[3].State = "queued"
|
||||||
|
|
||||||
|
dls.StartQueued(0)
|
||||||
time.Sleep(time.Millisecond * 100)
|
time.Sleep(time.Millisecond * 100)
|
||||||
if q.Downloads[0].State != STATE_COMPLETE {
|
|
||||||
t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
|
// they should all be going
|
||||||
t.Log(q.String())
|
if dls[0].State == "queued" || dls[1].State == "queued" || dls[2].State == "queued" || dls[3].State == "queued" {
|
||||||
}
|
t.Error("none should be queued")
|
||||||
if q.Downloads[1].State != STATE_DOWNLOADING {
|
|
||||||
t.Errorf("#2 is not downloading - %s instead", q.Downloads[1].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[2].State != STATE_QUEUED {
|
|
||||||
t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[3].State != STATE_COMPLETE {
|
|
||||||
t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// but if we allow two per domain, the other queued one will start
|
// reset them all
|
||||||
q.startQueued(2)
|
dls[0].State = "queued"
|
||||||
|
dls[1].State = "queued"
|
||||||
|
dls[2].State = "queued"
|
||||||
|
dls[3].State = "queued"
|
||||||
|
|
||||||
|
dls.StartQueued(2)
|
||||||
time.Sleep(time.Millisecond * 100)
|
time.Sleep(time.Millisecond * 100)
|
||||||
if q.Downloads[0].State != STATE_COMPLETE {
|
|
||||||
t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
|
// first two should be running, third not (same domain) and 4th running (different domain)
|
||||||
t.Log(q.String())
|
if dls[0].State == "queued" || dls[1].State == "queued" || dls[2].State != "queued" || dls[3].State == "queued" {
|
||||||
}
|
t.Error("incorrect queued")
|
||||||
if q.Downloads[1].State != STATE_DOWNLOADING {
|
|
||||||
t.Errorf("#2 is not downloading - %s instead", q.Downloads[1].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[2].State != STATE_DOWNLOADING {
|
|
||||||
t.Errorf("#3 is not downloading - %s instead", q.Downloads[2].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
if q.Downloads[3].State != STATE_COMPLETE {
|
|
||||||
t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
|
|
||||||
t.Log(q.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdateMetadataPlaylist(t *testing.T) {
|
|
||||||
|
|
||||||
output := `
|
|
||||||
start of log...
|
|
||||||
[download] Downloading playlist: nice_user
|
|
||||||
[RedGifsUser] nice_user: Downloading JSON metadata page 1
|
|
||||||
[RedGifsUser] nice_user: Downloading JSON metadata page 2
|
|
||||||
[RedGifsUser] nice_user: Downloading JSON metadata page 3
|
|
||||||
[RedGifsUser] nice_user: Downloading JSON metadata page 4
|
|
||||||
[RedGifsUser] nice_user: Downloading JSON metadata page 5
|
|
||||||
[RedGifsUser] nice_user: Downloading JSON metadata page 6
|
|
||||||
[info] Writing playlist metadata as JSON to: nice_user [nice_user].info.json
|
|
||||||
[RedGifsUser] playlist nice_user: Downloading 3 videos
|
|
||||||
[download] Downloading video 1 of 3
|
|
||||||
[info] wrongpreciouschrysomelid: Downloading 1 format(s): hd
|
|
||||||
[info] Writing video metadata as JSON to: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].info.json
|
|
||||||
[download] Destination: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].mp4
|
|
||||||
[download] 0.0% of 4.96MiB at Unknown speed ETA Unknown
|
|
||||||
[download] 0.1% of 4.96MiB at 1.76MiB/s ETA 00:02
|
|
||||||
[download] 20.1% of 4.96MiB at 7.28MiB/s ETA 00:00
|
|
||||||
[download] 40.3% of 4.96MiB at 10.06MiB/s ETA 00:00
|
|
||||||
[download] 80.6% of 4.96MiB at 14.93MiB/s ETA 00:00
|
|
||||||
[download] 100% of 4.96MiB at 17.33MiB/s ETA 00:00
|
|
||||||
[download] 100% of 4.96MiB in 00:00
|
|
||||||
[download] Downloading video 2 of 3
|
|
||||||
[info] silentnaughtyborzoi: Downloading 1 format(s): hd
|
|
||||||
[info] Writing video metadata as JSON to: Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].info.json
|
|
||||||
[download] Destination: Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].mp4
|
|
||||||
[download] 0.0% of 5.81MiB at 896.03KiB/s ETA 00:06
|
|
||||||
[download] 0.1% of 5.81MiB at 1.28MiB/s ETA 00:04
|
|
||||||
[download] 0.1% of 5.81MiB at 1.59MiB/s ETA 00:03
|
|
||||||
[download] 34.4% of 5.81MiB at 9.90MiB/s ETA 00:00
|
|
||||||
[download] 68.8% of 5.81MiB at 12.49MiB/s ETA 00:00
|
|
||||||
[download] 100% of 5.81MiB at 15.77MiB/s ETA 00:00
|
|
||||||
[download] 100% of 5.81MiB in 00:00
|
|
||||||
[download] Downloading video 3 of 3
|
|
||||||
[info] mammothremarkablewhooper: Downloading 1 format(s): hd
|
|
||||||
[info] Writing video metadata as JSON to: Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].info.json
|
|
||||||
[download] Destination: Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].mp4
|
|
||||||
[download] 0.0% of 2.89MiB at Unknown speed ETA Unknown
|
|
||||||
[download] 0.1% of 2.89MiB at 1.77MiB/s ETA 00:01
|
|
||||||
[download] 0.2% of 2.89MiB at 2.26MiB/s ETA 00:01
|
|
||||||
[download] 34.5% of 2.89MiB at 8.23MiB/s ETA 00:00
|
|
||||||
[download] 69.1% of 2.89MiB at 11.63MiB/s ETA 00:00
|
|
||||||
[download] 100% of 2.89MiB at 14.25MiB/s ETA 00:00
|
|
||||||
[download] 100% of 2.89MiB in 00:00
|
|
||||||
[info] Writing updated playlist metadata as JSON to: nice_user [nice_user].info.json
|
|
||||||
[download] Finished downloading playlist: nice_user
|
|
||||||
`
|
|
||||||
newD := Download{}
|
|
||||||
|
|
||||||
lines := strings.Split(output, "\n")
|
|
||||||
for _, l := range lines {
|
|
||||||
// t.Log(l)
|
|
||||||
newD.updateMetadata(l)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(newD.Files) != 3 {
|
|
||||||
t.Errorf("%d files, not 3", len(newD.Files))
|
|
||||||
} else {
|
|
||||||
if newD.Files[0] != "Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].mp4" {
|
|
||||||
t.Error("Wrong 1st file")
|
|
||||||
}
|
|
||||||
if newD.Files[1] != "Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].mp4" {
|
|
||||||
t.Error("Wrong 2nd file")
|
|
||||||
}
|
|
||||||
if newD.Files[2] != "Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].mp4" {
|
|
||||||
t.Error("Wrong 3rd file")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if newD.PlaylistTotal != 3 {
|
|
||||||
t.Errorf("playlist has total %d should be 3", newD.PlaylistTotal)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdateMetadataSingle(t *testing.T) {
|
|
||||||
|
|
||||||
output := `
|
|
||||||
[youtube] 2WoDQBhJCVQ: Downloading webpage
|
|
||||||
[youtube] 2WoDQBhJCVQ: Downloading android player API JSON
|
|
||||||
[info] 2WoDQBhJCVQ: Downloading 1 format(s): 137+140
|
|
||||||
[info] Writing video metadata as JSON to: The Greatest Shot In Television [2WoDQBhJCVQ].info.json
|
|
||||||
[debug] Invoking hlsnative downloader on "https://example.org/urls/1.2.3.4%
|
|
||||||
[download] 0.0% of 12.82MiB at 510.94KiB/s ETA 00:26
|
|
||||||
[download] 0.0% of 12.82MiB at 966.50KiB/s ETA 00:13
|
|
||||||
[download] 0.1% of 12.82MiB at 1.54MiB/s ETA 00:08
|
|
||||||
[download] 0.1% of 12.82MiB at 2.75MiB/s ETA 00:04
|
|
||||||
[download] 0.2% of 12.82MiB at 1.30MiB/s ETA 00:09
|
|
||||||
[download] 77.5% of 12.82MiB at 2.54MiB/s ETA 00:01
|
|
||||||
[download] 79.4% of 12.82MiB at 3.89MiB/s ETA 00:00
|
|
||||||
[download] 83.3% of 12.82MiB at 6.44MiB/s ETA 00:00
|
|
||||||
[download] 91.1% of 12.82MiB at 10.28MiB/s ETA 00:00
|
|
||||||
[download] 100% of 12.82MiB at 12.77MiB/s ETA 00:00
|
|
||||||
[download] 100% of 12.82MiB in 00:01
|
|
||||||
[download] Destination: The Greatest Shot In Television [2WoDQBhJCVQ].f140.m4a
|
|
||||||
[download] 0.1% of 1.10MiB at 286.46KiB/s ETA 00:03
|
|
||||||
[download] 0.3% of 1.10MiB at 716.49KiB/s ETA 00:01
|
|
||||||
[download] 0.6% of 1.10MiB at 1.42MiB/s ETA 00:00
|
|
||||||
[download] 91.0% of 1.10MiB at 6.67MiB/s ETA 00:00
|
|
||||||
[download] 100% of 1.10MiB at 7.06MiB/s ETA 00:00
|
|
||||||
[download] 100% of 1.10MiB in 00:00
|
|
||||||
[Merger] Merging formats into "The Greatest Shot In Television [2WoDQBhJCVQ].mp4"
|
|
||||||
Deleting original file The Greatest Shot In Television [2WoDQBhJCVQ].f137.mp4 (pass -k to keep)
|
|
||||||
Deleting original file The Greatest Shot In Television [2WoDQBhJCVQ].f140.m4a (pass -k to keep)
|
|
||||||
`
|
|
||||||
newD := Download{}
|
|
||||||
|
|
||||||
lines := strings.Split(output, "\n")
|
|
||||||
for _, l := range lines {
|
|
||||||
// t.Log(l)
|
|
||||||
newD.updateMetadata(l)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(newD.Files) != 1 {
|
|
||||||
t.Errorf("%d files, not 1", len(newD.Files))
|
|
||||||
} else {
|
|
||||||
if newD.Files[0] != "The Greatest Shot In Television [2WoDQBhJCVQ].mp4" {
|
|
||||||
t.Error("Wrong 1st file")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if newD.PlaylistTotal != 0 {
|
|
||||||
t.Error("playlist detected but should not be")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,41 +0,0 @@
|
|||||||
//go:build testdata
|
|
||||||
|
|
||||||
package download
|
|
||||||
|
|
||||||
import "github.com/tardisx/gropple/config"
|
|
||||||
|
|
||||||
func (m *Manager) AddStressTestData(c *config.ConfigService) {
|
|
||||||
|
|
||||||
urls := []string{
|
|
||||||
"https://www.youtube.com/watch?v=qG_rRkuGBW8",
|
|
||||||
"https://www.youtube.com/watch?v=ZUzhZpQAU40",
|
|
||||||
"https://www.youtube.com/watch?v=kVxM3eRWGak",
|
|
||||||
"https://www.youtube.com/watch?v=pl-y9869y0w",
|
|
||||||
"https://www.youtube.com/watch?v=Uw4NEPE4l3A",
|
|
||||||
"https://www.youtube.com/watch?v=2RF0lcTuuYE",
|
|
||||||
"https://www.youtube.com/watch?v=lymwNQY0dus",
|
|
||||||
"https://www.youtube.com/watch?v=NTc-I4Z_duc",
|
|
||||||
"https://www.youtube.com/watch?v=wNSm1TJ84Ac",
|
|
||||||
"https://www.youtube.com/watch?v=tyixMpuGEL8",
|
|
||||||
"https://www.youtube.com/watch?v=VnxbkH_3E_4",
|
|
||||||
"https://www.youtube.com/watch?v=VStscvYLYLs",
|
|
||||||
"https://www.youtube.com/watch?v=vYMiSz-WlEY",
|
|
||||||
|
|
||||||
"https://vimeo.com/786570322",
|
|
||||||
"https://vimeo.com/783453809",
|
|
||||||
|
|
||||||
"https://www.gamespot.com/videos/survival-fps-how-metro-2033-solidified-a-subgenre/2300-6408243/",
|
|
||||||
"https://www.gamespot.com/videos/dirt-3-right-back-where-you-started-gameplay-movie/2300-6314712/",
|
|
||||||
"https://www.gamespot.com/videos/the-b-list-driver-san-francisco/2300-6405593/",
|
|
||||||
|
|
||||||
"https://www.imdb.com/video/vi1914750745/?listId=ls053181649&ref_=hm_hp_i_hero-video-1_1",
|
|
||||||
"https://www.imdb.com/video/vi3879585561/?listId=ls053181649&ref_=vp_pl_ap_6",
|
|
||||||
"https://www.imdb.com/video/vi54445849/?listId=ls053181649&ref_=vp_nxt_btn",
|
|
||||||
}
|
|
||||||
for _, u := range urls {
|
|
||||||
d := NewDownload(u, c.Config)
|
|
||||||
d.DownloadProfile = *c.Config.ProfileCalled("standard video")
|
|
||||||
m.AddDownload(d)
|
|
||||||
m.Queue(d)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
15
go.mod
15
go.mod
@ -1,18 +1,9 @@
|
|||||||
module github.com/tardisx/gropple
|
module github.com/tardisx/gropple
|
||||||
|
|
||||||
go 1.23.0
|
go 1.16
|
||||||
|
|
||||||
toolchain go1.24.1
|
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/gorilla/mux v1.8.1
|
github.com/gorilla/mux v1.8.0
|
||||||
github.com/stretchr/testify v1.9.0
|
golang.org/x/mod v0.5.1
|
||||||
golang.org/x/mod v0.24.0
|
|
||||||
gopkg.in/yaml.v2 v2.4.0
|
gopkg.in/yaml.v2 v2.4.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
|
||||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
|
||||||
)
|
|
||||||
|
|||||||
27
go.sum
27
go.sum
@ -1,16 +1,19 @@
|
|||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
|
||||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
|
||||||
|
|||||||
339
main.go
339
main.go
@ -1,41 +1,48 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"flag"
|
"embed"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"html/template"
|
||||||
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
"github.com/gorilla/mux"
|
||||||
"github.com/tardisx/gropple/config"
|
"github.com/tardisx/gropple/config"
|
||||||
"github.com/tardisx/gropple/download"
|
"github.com/tardisx/gropple/download"
|
||||||
v "github.com/tardisx/gropple/version"
|
"github.com/tardisx/gropple/version"
|
||||||
"github.com/tardisx/gropple/web"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var downloads download.Downloads
|
||||||
version = "dev"
|
var downloadId = 0
|
||||||
)
|
var configService *config.ConfigService
|
||||||
|
|
||||||
|
var versionInfo = version.Info{CurrentVersion: "v0.5.4"}
|
||||||
|
|
||||||
|
//go:embed web
|
||||||
|
var webFS embed.FS
|
||||||
|
|
||||||
|
type successResponse struct {
|
||||||
|
Success bool `json:"success"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type errorResponse struct {
|
||||||
|
Success bool `json:"success"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
versionInfo := &v.Manager{
|
log.Printf("Starting gropple %s - https://github.com/tardisx/gropple", versionInfo.CurrentVersion)
|
||||||
// version from goreleaser has no leading 'v', even if the tag does
|
|
||||||
VersionInfo: v.Info{CurrentVersion: "v" + version},
|
|
||||||
}
|
|
||||||
log.Printf("Starting gropple %s - https://github.com/tardisx/gropple", versionInfo.GetInfo().CurrentVersion)
|
|
||||||
|
|
||||||
var configPath string
|
|
||||||
flag.StringVar(&configPath, "config-path", "", "path to config file")
|
|
||||||
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
configService := &config.ConfigService{}
|
|
||||||
if configPath != "" {
|
|
||||||
configService.ConfigPath = configPath
|
|
||||||
} else {
|
|
||||||
configService.DetermineConfigDir()
|
|
||||||
}
|
|
||||||
|
|
||||||
|
configService = &config.ConfigService{}
|
||||||
|
configService.DetermineConfigDir()
|
||||||
exists, err := configService.ConfigFileExists()
|
exists, err := configService.ConfigFileExists()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
@ -51,13 +58,24 @@ func main() {
|
|||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
log.Printf("Configuration loaded from %s", configService.ConfigPath)
|
log.Printf("Configuration loaded from %s", configService.ConfigPath)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// create the download manager
|
r := mux.NewRouter()
|
||||||
downloadManager := &download.Manager{MaxPerDomain: configService.Config.Server.MaximumActiveDownloads}
|
r.HandleFunc("/", homeHandler)
|
||||||
|
r.HandleFunc("/static/{filename}", staticHandler)
|
||||||
|
r.HandleFunc("/config", configHandler)
|
||||||
|
r.HandleFunc("/fetch", fetchHandler)
|
||||||
|
r.HandleFunc("/fetch/{id}", fetchHandler)
|
||||||
|
|
||||||
// create the web handlers
|
// info for the list
|
||||||
r := web.CreateRoutes(configService, downloadManager, versionInfo)
|
r.HandleFunc("/rest/fetch", fetchInfoRESTHandler)
|
||||||
|
// info for one, including update
|
||||||
|
r.HandleFunc("/rest/fetch/{id}", fetchInfoOneRESTHandler)
|
||||||
|
r.HandleFunc("/rest/version", versionRESTHandler)
|
||||||
|
r.HandleFunc("/rest/config", configRESTHandler)
|
||||||
|
|
||||||
|
http.Handle("/", r)
|
||||||
|
|
||||||
srv := &http.Server{
|
srv := &http.Server{
|
||||||
Handler: r,
|
Handler: r,
|
||||||
@ -70,22 +88,273 @@ func main() {
|
|||||||
// check for a new version every 4 hours
|
// check for a new version every 4 hours
|
||||||
go func() {
|
go func() {
|
||||||
for {
|
for {
|
||||||
err := versionInfo.UpdateGitHubVersion()
|
versionInfo.UpdateGitHubVersion()
|
||||||
if err != nil {
|
|
||||||
log.Printf("could not get version info: %s", err)
|
|
||||||
}
|
|
||||||
time.Sleep(time.Hour * 4)
|
time.Sleep(time.Hour * 4)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// start downloading queued downloads when slots available, and clean up
|
// start downloading queued downloads when slots available, and clean up
|
||||||
// old entries
|
// old entries
|
||||||
go downloadManager.ManageQueue()
|
go func() {
|
||||||
|
for {
|
||||||
// add testdata if compiled with the '-tags testdata' flag
|
downloads.StartQueued(configService.Config.Server.MaximumActiveDownloads)
|
||||||
downloadManager.AddStressTestData(configService)
|
downloads = downloads.Cleanup()
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
log.Printf("Visit %s for details on installing the bookmarklet and to check status", configService.Config.Server.Address)
|
log.Printf("Visit %s for details on installing the bookmarklet and to check status", configService.Config.Server.Address)
|
||||||
log.Fatal(srv.ListenAndServe())
|
log.Fatal(srv.ListenAndServe())
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// versionRESTHandler returns the version information, if we have up-to-date info from github
|
||||||
|
func versionRESTHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if versionInfo.GithubVersionFetched {
|
||||||
|
b, _ := json.Marshal(versionInfo)
|
||||||
|
w.Write(b)
|
||||||
|
} else {
|
||||||
|
w.WriteHeader(400)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// homeHandler returns the main index page
|
||||||
|
func homeHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
|
||||||
|
bookmarkletURL := fmt.Sprintf("javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('%s/fetch?url=',window.location,'yourform','width=%d,height=%d'));", configService.Config.Server.Address, configService.Config.UI.PopupWidth, configService.Config.UI.PopupHeight)
|
||||||
|
|
||||||
|
t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/menu.tmpl", "web/index.html")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
type Info struct {
|
||||||
|
Downloads []*download.Download
|
||||||
|
BookmarkletURL template.URL
|
||||||
|
Config *config.Config
|
||||||
|
}
|
||||||
|
|
||||||
|
info := Info{
|
||||||
|
Downloads: downloads,
|
||||||
|
BookmarkletURL: template.URL(bookmarkletURL),
|
||||||
|
Config: configService.Config,
|
||||||
|
}
|
||||||
|
|
||||||
|
err = t.ExecuteTemplate(w, "layout", info)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// staticHandler handles requests for static files
|
||||||
|
func staticHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
vars := mux.Vars(r)
|
||||||
|
|
||||||
|
filename := vars["filename"]
|
||||||
|
if strings.Index(filename, ".js") == len(filename)-3 {
|
||||||
|
f, err := webFS.Open("web/" + filename)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("error accessing %s - %v", filename, err)
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
io.Copy(w, f)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
// configHandler returns the configuration page
|
||||||
|
func configHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
|
||||||
|
t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/menu.tmpl", "web/config.html")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = t.ExecuteTemplate(w, "layout", nil)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// configRESTHandler handles both reading and writing of the configuration
|
||||||
|
func configRESTHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
|
if r.Method == "POST" {
|
||||||
|
log.Printf("Updating config")
|
||||||
|
b, err := io.ReadAll(r.Body)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
err = configService.Config.UpdateFromJSON(b)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
errorRes := errorResponse{Success: false, Error: err.Error()}
|
||||||
|
errorResB, _ := json.Marshal(errorRes)
|
||||||
|
w.WriteHeader(400)
|
||||||
|
w.Write(errorResB)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
configService.WriteConfig()
|
||||||
|
}
|
||||||
|
b, _ := json.Marshal(configService.Config)
|
||||||
|
w.Write(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
vars := mux.Vars(r)
|
||||||
|
idString := vars["id"]
|
||||||
|
if idString != "" {
|
||||||
|
id, err := strconv.Atoi(idString)
|
||||||
|
if err != nil {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// find the download
|
||||||
|
var thisDownload *download.Download
|
||||||
|
for _, dl := range downloads {
|
||||||
|
if dl.Id == id {
|
||||||
|
thisDownload = dl
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if thisDownload == nil {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.Method == "POST" {
|
||||||
|
|
||||||
|
type updateRequest struct {
|
||||||
|
Action string `json:"action"`
|
||||||
|
Profile string `json:"profile"`
|
||||||
|
}
|
||||||
|
|
||||||
|
thisReq := updateRequest{}
|
||||||
|
|
||||||
|
b, err := io.ReadAll(r.Body)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = json.Unmarshal(b, &thisReq)
|
||||||
|
if err != nil {
|
||||||
|
errorRes := errorResponse{Success: false, Error: err.Error()}
|
||||||
|
errorResB, _ := json.Marshal(errorRes)
|
||||||
|
w.WriteHeader(400)
|
||||||
|
w.Write(errorResB)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if thisReq.Action == "start" {
|
||||||
|
// find the profile they asked for
|
||||||
|
profile := configService.Config.ProfileCalled(thisReq.Profile)
|
||||||
|
if profile == nil {
|
||||||
|
panic("bad profile name?")
|
||||||
|
}
|
||||||
|
// set the profile
|
||||||
|
thisDownload.DownloadProfile = *profile
|
||||||
|
|
||||||
|
thisDownload.Queue()
|
||||||
|
succRes := successResponse{Success: true, Message: "download started"}
|
||||||
|
succResB, _ := json.Marshal(succRes)
|
||||||
|
w.Write(succResB)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if thisReq.Action == "stop" {
|
||||||
|
thisDownload.Stop()
|
||||||
|
succRes := successResponse{Success: true, Message: "download stopped"}
|
||||||
|
succResB, _ := json.Marshal(succRes)
|
||||||
|
w.Write(succResB)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// just a get, return the object
|
||||||
|
b, _ := json.Marshal(thisDownload)
|
||||||
|
w.Write(b)
|
||||||
|
return
|
||||||
|
} else {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchInfoRESTHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
b, _ := json.Marshal(downloads)
|
||||||
|
w.Write(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
|
// if they refreshed the popup, just load the existing object, don't
|
||||||
|
// create a new one
|
||||||
|
vars := mux.Vars(r)
|
||||||
|
idString := vars["id"]
|
||||||
|
|
||||||
|
idInt, err := strconv.ParseInt(idString, 10, 32)
|
||||||
|
if err == nil && idInt > 0 {
|
||||||
|
for _, dl := range downloads {
|
||||||
|
if dl.Id == int(idInt) {
|
||||||
|
t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
templateData := map[string]interface{}{"dl": dl, "config": configService.Config, "canStop": download.CanStopDownload}
|
||||||
|
|
||||||
|
err = t.ExecuteTemplate(w, "layout", templateData)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
query := r.URL.Query()
|
||||||
|
url, present := query["url"]
|
||||||
|
|
||||||
|
if !present {
|
||||||
|
w.WriteHeader(400)
|
||||||
|
fmt.Fprint(w, "No url supplied")
|
||||||
|
return
|
||||||
|
} else {
|
||||||
|
|
||||||
|
// check the URL for a sudden but inevitable betrayal
|
||||||
|
if strings.Contains(url[0], configService.Config.Server.Address) {
|
||||||
|
w.WriteHeader(400)
|
||||||
|
fmt.Fprint(w, "you mustn't gropple your gropple :-)")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// create the record
|
||||||
|
newDownload := download.NewDownload(configService.Config, url[0])
|
||||||
|
downloads = append(downloads, newDownload)
|
||||||
|
// XXX atomic ^^
|
||||||
|
|
||||||
|
newDownload.Log = append(newDownload.Log, "start of log...")
|
||||||
|
|
||||||
|
// go func() {
|
||||||
|
// newDownload.Begin()
|
||||||
|
// }()
|
||||||
|
|
||||||
|
t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
templateData := map[string]interface{}{"dl": newDownload, "config": configService.Config, "canStop": download.CanStopDownload}
|
||||||
|
|
||||||
|
err = t.ExecuteTemplate(w, "layout", templateData)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -8,7 +8,6 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"sync"
|
|
||||||
|
|
||||||
"golang.org/x/mod/semver"
|
"golang.org/x/mod/semver"
|
||||||
)
|
)
|
||||||
@ -20,23 +19,8 @@ type Info struct {
|
|||||||
GithubVersionFetched bool `json:"-"`
|
GithubVersionFetched bool `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Manager struct {
|
func (i *Info) UpdateGitHubVersion() error {
|
||||||
VersionInfo Info
|
i.GithubVersionFetched = false
|
||||||
lock sync.Mutex
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Manager) GetInfo() Info {
|
|
||||||
m.lock.Lock()
|
|
||||||
defer m.lock.Unlock()
|
|
||||||
|
|
||||||
return m.VersionInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Manager) UpdateGitHubVersion() error {
|
|
||||||
m.lock.Lock()
|
|
||||||
m.VersionInfo.GithubVersionFetched = false
|
|
||||||
m.lock.Unlock()
|
|
||||||
|
|
||||||
versionUrl := "https://api.github.com/repos/tardisx/gropple/releases"
|
versionUrl := "https://api.github.com/repos/tardisx/gropple/releases"
|
||||||
resp, err := http.Get(versionUrl)
|
resp, err := http.Get(versionUrl)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -67,30 +51,27 @@ func (m *Manager) UpdateGitHubVersion() error {
|
|||||||
return errors.New("no releases found")
|
return errors.New("no releases found")
|
||||||
}
|
}
|
||||||
|
|
||||||
m.lock.Lock()
|
i.GithubVersion = releases[0].Name
|
||||||
|
|
||||||
defer m.lock.Unlock()
|
|
||||||
m.VersionInfo.GithubVersion = releases[0].Name
|
|
||||||
m.VersionInfo.GithubVersionFetched = true
|
|
||||||
m.VersionInfo.UpgradeAvailable = m.canUpgrade()
|
|
||||||
|
|
||||||
|
i.GithubVersionFetched = true
|
||||||
|
i.UpgradeAvailable = i.canUpgrade()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Manager) canUpgrade() bool {
|
func (i *Info) canUpgrade() bool {
|
||||||
if !m.VersionInfo.GithubVersionFetched {
|
if !i.GithubVersionFetched {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
if !semver.IsValid(m.VersionInfo.CurrentVersion) {
|
if !semver.IsValid(i.CurrentVersion) {
|
||||||
log.Printf("current version %s is invalid", m.VersionInfo.CurrentVersion)
|
log.Printf("current version %s is invalid", i.CurrentVersion)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !semver.IsValid(m.VersionInfo.GithubVersion) {
|
if !semver.IsValid(i.GithubVersion) {
|
||||||
log.Printf("github version %s is invalid", m.VersionInfo.GithubVersion)
|
log.Printf("github version %s is invalid", i.GithubVersion)
|
||||||
}
|
}
|
||||||
|
|
||||||
if semver.Compare(m.VersionInfo.CurrentVersion, m.VersionInfo.GithubVersion) == -1 {
|
if semver.Compare(i.CurrentVersion, i.GithubVersion) == -1 {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
|
|||||||
@ -7,17 +7,11 @@
|
|||||||
<p class="error" x-show="error_message" x-transition.duration.500ms x-text="error_message"></p>
|
<p class="error" x-show="error_message" x-transition.duration.500ms x-text="error_message"></p>
|
||||||
<p class="success" x-show="success_message" x-transition.duration.500ms x-text="success_message"></p>
|
<p class="success" x-show="success_message" x-transition.duration.500ms x-text="success_message"></p>
|
||||||
|
|
||||||
<p>Note: changes are not saved until the "Save Config" button is pressed.</p>
|
<p>Note: changes are not saved until the "Save Config" button is pressed at the bottom of the page.</p>
|
||||||
|
|
||||||
<div class="pure-g">
|
|
||||||
<div class="pure-u-1">
|
|
||||||
<button class="button-small pure-button button-small pure-button-primary" @click="save_config();" href="#">Save Config</button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="pure-g">
|
<div class="pure-g">
|
||||||
|
|
||||||
<div class="pure-u-lg-1-3 pure-u-1 l-box">
|
<div class="pure-u-md-1-2 pure-u-1 l-box">
|
||||||
|
|
||||||
<form class="pure-form pure-form-stacked gropple-config">
|
<form class="pure-form pure-form-stacked gropple-config">
|
||||||
<fieldset>
|
<fieldset>
|
||||||
@ -37,7 +31,7 @@
|
|||||||
|
|
||||||
<label for="config-server-downloadpath">Download path</label>
|
<label for="config-server-downloadpath">Download path</label>
|
||||||
<input type="text" id="config-server-downloadpath" placeholder="path" class="input-long" x-model="config.server.download_path" />
|
<input type="text" id="config-server-downloadpath" placeholder="path" class="input-long" x-model="config.server.download_path" />
|
||||||
<span class="pure-form-message">The default path on the server to download files to.</span>
|
<span class="pure-form-message">The path on the server to download files to.</span>
|
||||||
|
|
||||||
<label for="config-server-max-downloads">Maximum active downloads per domain</label>
|
<label for="config-server-max-downloads">Maximum active downloads per domain</label>
|
||||||
<input type="text" id="config-server-max-downloads" placeholder="2" class="input-long" x-model.number="config.server.maximum_active_downloads_per_domain" />
|
<input type="text" id="config-server-max-downloads" placeholder="2" class="input-long" x-model.number="config.server.maximum_active_downloads_per_domain" />
|
||||||
@ -59,7 +53,7 @@
|
|||||||
</form>
|
</form>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
<div class="pure-u-lg-1-3 pure-u-1 l-box">
|
<div class="pure-u-md-1-2 pure-u-1 l-box">
|
||||||
<form class="pure-form gropple-config">
|
<form class="pure-form gropple-config">
|
||||||
<fieldset>
|
<fieldset>
|
||||||
|
|
||||||
@ -78,16 +72,13 @@
|
|||||||
</label>
|
</label>
|
||||||
|
|
||||||
<input type="text" x-bind:id="'config-profiles-'+i+'-name'" class="input-long" placeholder="name" x-model="profile.name" />
|
<input type="text" x-bind:id="'config-profiles-'+i+'-name'" class="input-long" placeholder="name" x-model="profile.name" />
|
||||||
<button class="button-small pure-button button-del" href="#" @click.prevent="config.profiles.splice(i, 1);;">delete profile</button>
|
<button class="pure-button button-del" href="#" @click.prevent="config.profiles.splice(i, 1);;">delete profile</button>
|
||||||
|
|
||||||
<span class="pure-form-message">The name of this profile. For your information only.</span>
|
<span class="pure-form-message">The name of this profile. For your information only.</span>
|
||||||
|
|
||||||
<label x-bind:for="'config-profiles-'+i+'-command'">Command to run</label>
|
<label x-bind:for="'config-profiles-'+i+'-command'">Command to run</label>
|
||||||
<input type="text" x-bind:id="'config-profiles-'+i+'-command'" class="input-long" placeholder="name" x-model="profile.command" />
|
<input type="text" x-bind:id="'config-profiles-'+i+'-command'" class="input-long" placeholder="name" x-model="profile.command" />
|
||||||
<span class="pure-form-message">Which command to run. Your path will be searched, or you can specify the full path here.
|
<span class="pure-form-message">Which command to run. Your path will be searched, or you can specify the full path here.</span>
|
||||||
If you are using gropple in portable mode and store the executables with the gropple executable, use a prefix of
|
|
||||||
<tt>./</tt>, for instance <tt>yt-dlp.exe</tt>.
|
|
||||||
</span>
|
|
||||||
|
|
||||||
|
|
||||||
<label>Arguments</label>
|
<label>Arguments</label>
|
||||||
@ -95,11 +86,11 @@
|
|||||||
<template x-for="(arg, j) in profile.args">
|
<template x-for="(arg, j) in profile.args">
|
||||||
<div>
|
<div>
|
||||||
<input type="text" x-bind:id="'config-profiles-'+i+'-arg-'+j" placeholder="arg" x-model="profile.args[j]" />
|
<input type="text" x-bind:id="'config-profiles-'+i+'-arg-'+j" placeholder="arg" x-model="profile.args[j]" />
|
||||||
<button class="button-small pure-button button-del" href="#" @click.prevent="profile.args.splice(j, 1);;">delete arg</button>
|
<button class="pure-button button-del" href="#" @click.prevent="profile.args.splice(j, 1);;">delete arg</button>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<button class="button-small pure-button button-add" href="#" @click.prevent="profile.args.push('');">add arg</button>
|
<button class="pure-button button-add" href="#" @click.prevent="profile.args.push('');">add arg</button>
|
||||||
<span class="pure-form-message">Arguments for the command. Note that the shell is not used, so there is no need to quote or escape arguments, including those with spaces.</span>
|
<span class="pure-form-message">Arguments for the command. Note that the shell is not used, so there is no need to quote or escape arguments, including those with spaces.</span>
|
||||||
|
|
||||||
<hr>
|
<hr>
|
||||||
@ -107,53 +98,15 @@
|
|||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<button class="button-small pure-button button-add" href="#" @click.prevent="config.profiles.push({name: 'new profile', command: 'youtube-dl', args: []});">add profile</button>
|
<button class="pure-button button-add" href="#" @click.prevent="config.profiles.push({name: 'new profile', command: 'youtube-dl', args: []});">add profile</button>
|
||||||
|
|
||||||
</fieldset>
|
</fieldset>
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="pure-u-lg-1-3 pure-u-1 l-box">
|
|
||||||
<form class="pure-form gropple-config">
|
|
||||||
<fieldset>
|
|
||||||
<legend>Download Options</legend>
|
|
||||||
<p>You can specify custom download options here. These are (optionally) selectable in addition
|
|
||||||
to the profile when starting a download. They append extra arguments to the downloader command.
|
|
||||||
The most common use is to specify a particular <tt>-o</tt> argument to <tt>yt-dlp</tt> to allow files to be downloaded
|
|
||||||
to a custom path.</p>
|
|
||||||
</p>
|
|
||||||
<template x-for="(download_option, i) in config.download_options">
|
|
||||||
<div>
|
|
||||||
<label x-bind:for="'config-download-option-'+i+'-name'">Name of option <span x-text="i+1"></span>
|
|
||||||
</label>
|
|
||||||
|
|
||||||
<input type="text" x-bind:id="'config-download-option-'+i+'-name'" class="input-long" placeholder="name" x-model="download_option.name" />
|
|
||||||
|
|
||||||
<span class="pure-form-message">The name of this option. For your information only.</span>
|
|
||||||
|
|
||||||
<label>Arguments</label>
|
|
||||||
|
|
||||||
<template x-for="(arg, j) in download_option.args">
|
|
||||||
<div>
|
|
||||||
<input type="text" x-bind:id="'config-download-option-'+i+'-arg-'+j" placeholder="arg" x-model="download_option.args[j]" />
|
|
||||||
<button class="button-small pure-button button-del" href="#" @click.prevent="download_option.args.splice(j, 1);;">delete arg</button>
|
|
||||||
</div>
|
|
||||||
</template>
|
|
||||||
<button class="button-small pure-button button-del" href="#" @click.prevent="config.download_options.splice(i, 1);">delete option</button>
|
|
||||||
|
|
||||||
<hr>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<button class="button-small pure-button button-add" href="#" @click.prevent="config.download_options.push({name: 'new option', args: ['-o', 'someting']});">add option</button>
|
|
||||||
|
|
||||||
</fieldset>
|
|
||||||
</form>
|
|
||||||
</div>
|
</div>
|
||||||
<div class="pure-g">
|
<div class="pure-g">
|
||||||
<div class="pure-u-1">
|
<div class="pure-u-1">
|
||||||
<button class="button-small pure-button button-small pure-button-primary" @click="save_config();" href="#">Save Config</button>
|
<button class="pure-button pure-button-primary" @click="save_config();" href="#">Save Config</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@ -166,7 +119,7 @@
|
|||||||
<script>
|
<script>
|
||||||
function config() {
|
function config() {
|
||||||
return {
|
return {
|
||||||
config: { server : {}, ui : {}, profiles: [], download_options: []},
|
config: { server : {}, ui : {}, profiles: [] },
|
||||||
error_message: '',
|
error_message: '',
|
||||||
success_message: '',
|
success_message: '',
|
||||||
|
|
||||||
@ -1,91 +0,0 @@
|
|||||||
{{ define "content" }}
|
|
||||||
|
|
||||||
{{ template "menu.tmpl" . }}
|
|
||||||
|
|
||||||
<div id="layout" class="pure-g pure-u-1" x-data="bulk_create()" >
|
|
||||||
|
|
||||||
<h1>Bulk upload</h1>
|
|
||||||
|
|
||||||
<p class="error" x-show="error_message" x-transition.duration.500ms x-text="error_message"></p>
|
|
||||||
<p class="success" x-show="success_message" x-transition.duration.500ms x-text="success_message"></p>
|
|
||||||
|
|
||||||
<p>Paste URLs here, one per line:</p>
|
|
||||||
|
|
||||||
<textarea x-model="urls" rows="20" cols="80">
|
|
||||||
</textarea>
|
|
||||||
|
|
||||||
<br><br>
|
|
||||||
|
|
||||||
<table class="pure-table" >
|
|
||||||
<tr>
|
|
||||||
<th>profile</th>
|
|
||||||
<td>
|
|
||||||
<select class="pure-input-1-2" x-model="profile_chosen">
|
|
||||||
<option value="">choose a profile</option>
|
|
||||||
{{ range $i := .config.DownloadProfiles }}
|
|
||||||
<option name="{{$i.Name}}">{{ $i.Name }}</option>
|
|
||||||
{{ end }}
|
|
||||||
</select>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>download option</th>
|
|
||||||
<td>
|
|
||||||
<select class="pure-input-1-2" x-model="download_option_chosen">
|
|
||||||
<option value="">no option</option>
|
|
||||||
{{ range $i := .config.DownloadOptions }}
|
|
||||||
<option name="{{$i.Name}}">{{ $i.Name }}</option>
|
|
||||||
{{ end }}
|
|
||||||
</select>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th> </th>
|
|
||||||
<td>
|
|
||||||
<button class="button-small pure-button" @click="start()">add to queue</button>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
|
|
||||||
</table>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{{ end }}
|
|
||||||
|
|
||||||
{{ define "js" }}
|
|
||||||
<script>
|
|
||||||
function bulk_create() {
|
|
||||||
return {
|
|
||||||
profile_chosen: "",
|
|
||||||
download_option_chosen: "",
|
|
||||||
urls: "",
|
|
||||||
error_message: "",
|
|
||||||
success_message: "",
|
|
||||||
start() {
|
|
||||||
let op = {
|
|
||||||
method: 'POST',
|
|
||||||
body: JSON.stringify({action: 'start', urls: this.urls, profile: this.profile_chosen, download_option: this.download_option_chosen}),
|
|
||||||
headers: { 'Content-Type': 'application/json' }
|
|
||||||
};
|
|
||||||
fetch('/bulk', op)
|
|
||||||
.then(response => response.json())
|
|
||||||
.then(response => {
|
|
||||||
console.log(response)
|
|
||||||
if (response.error) {
|
|
||||||
this.error_message = response.error;
|
|
||||||
this.success_message = '';
|
|
||||||
document.body.scrollTop = document.documentElement.scrollTop = 0;
|
|
||||||
} else {
|
|
||||||
this.error_message = '';
|
|
||||||
this.success_message = response.message;
|
|
||||||
this.urls = '';
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
{{ end }}
|
|
||||||
@ -1,74 +0,0 @@
|
|||||||
{{ define "content" }}
|
|
||||||
<div id="layout" class="pure-g pure-u-1" x-data="popup_create()" >
|
|
||||||
|
|
||||||
<h2>Download create</h2>
|
|
||||||
<p>URL: <tt>{{ .url }}</tt></p>
|
|
||||||
|
|
||||||
<p class="error" x-show="error_message" x-transition.duration.500ms x-text="error_message"></p>
|
|
||||||
|
|
||||||
<table class="pure-table" >
|
|
||||||
<tr>
|
|
||||||
<th>profile</th>
|
|
||||||
<td>
|
|
||||||
<select class="pure-input-1-2" x-model="profile_chosen">
|
|
||||||
<option value="">choose a profile</option>
|
|
||||||
{{ range $i := .config.DownloadProfiles }}
|
|
||||||
<option name="{{$i.Name}}">{{ $i.Name }}</option>
|
|
||||||
{{ end }}
|
|
||||||
</select>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th>download option</th>
|
|
||||||
<td>
|
|
||||||
<select class="pure-input-1-2" x-model="download_option_chosen">
|
|
||||||
<option value="">no option</option>
|
|
||||||
{{ range $i := .config.DownloadOptions }}
|
|
||||||
<option name="{{$i.Name}}">{{ $i.Name }}</option>
|
|
||||||
{{ end }}
|
|
||||||
</select>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<th> </th>
|
|
||||||
<td>
|
|
||||||
<button class="button-small pure-button" @click="start()">start download</button>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
{{ end }}
|
|
||||||
{{ define "js" }}
|
|
||||||
<script>
|
|
||||||
function popup_create() {
|
|
||||||
return {
|
|
||||||
profile_chosen: "",
|
|
||||||
download_option_chosen: "",
|
|
||||||
error_message: "",
|
|
||||||
start() {
|
|
||||||
let op = {
|
|
||||||
method: 'POST',
|
|
||||||
body: JSON.stringify({action: 'start', url: '{{ .url }}', profile: this.profile_chosen, download_option: this.download_option_chosen}),
|
|
||||||
headers: { 'Content-Type': 'application/json' }
|
|
||||||
};
|
|
||||||
fetch('/fetch', op)
|
|
||||||
.then(response => response.json())
|
|
||||||
.then(response => {
|
|
||||||
console.log(response)
|
|
||||||
if (response.error) {
|
|
||||||
this.error_message = response.error;
|
|
||||||
this.success_message = '';
|
|
||||||
document.body.scrollTop = document.documentElement.scrollTop = 0;
|
|
||||||
} else {
|
|
||||||
this.error_message = '';
|
|
||||||
console.log(response.location)
|
|
||||||
window.location = response.location
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
{{ end }}
|
|
||||||
|
|
||||||
@ -24,43 +24,16 @@
|
|||||||
<table class="pure-table">
|
<table class="pure-table">
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
<th>id</th>
|
<th>id</th><th>filename</th><th>url</th><th>show</th><th>state</th><th>percent</th><th>eta</th><th>finished</th>
|
||||||
<th>filename</th>
|
|
||||||
<th>url</th>
|
|
||||||
<th>state</th>
|
|
||||||
<th>percent</th>
|
|
||||||
<th>eta</th>
|
|
||||||
<th>finished</th>
|
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
<template x-for="item in items">
|
<template x-for="item in items">
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<td x-text="item.id"></td>
|
||||||
<a class="int-link" @click="show_popup(item)" href="#">
|
<td x-text="item.files"></td>
|
||||||
<span x-text="item.id">
|
<td><a class="int-link" x-bind:href="item.url">↗</a></td>
|
||||||
</a>
|
<td><a class="int-link" @click="show_popup(item)" href="#">📄</a></td>
|
||||||
</td>
|
|
||||||
<td>
|
|
||||||
<span x-show="item.files && item.files.length == 1">
|
|
||||||
<span class="filelist" x-text="item.files[0]"></span>
|
|
||||||
</span>
|
|
||||||
<span x-data="{open: false}" x-show="item.files && item.files.length > 1">
|
|
||||||
<span class="filelist" x-text="item.files.length + ' files...'"></span>
|
|
||||||
<button class="pure-button button-small" @click="open = ! open" x-text="open ? 'hide' : 'show'"></button>
|
|
||||||
<div x-show="open" x-transition>
|
|
||||||
<ul class="filelist">
|
|
||||||
<template x-for="file in item.files">
|
|
||||||
<li x-text="file"></li>
|
|
||||||
</template>
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
</span>
|
|
||||||
<span class="filelist" x-show="! item.files || item.files.length == 0"
|
|
||||||
x-text="'fetching ' + item.url + '...'">
|
|
||||||
</span>
|
|
||||||
</td>
|
|
||||||
<td><a class="int-link" x-bind:href="item.url">🔗</a></td>
|
|
||||||
<td :class="'state-'+item.state" x-text="item.state"></td>
|
<td :class="'state-'+item.state" x-text="item.state"></td>
|
||||||
<td x-text="item.percent"></td>
|
<td x-text="item.percent"></td>
|
||||||
<td x-text="item.eta"></td>
|
<td x-text="item.eta"></td>
|
||||||
@ -70,6 +43,8 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
|
|
||||||
|
{{ range $k, $v := .Downloads }}
|
||||||
|
{{ end }}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
@ -79,7 +54,7 @@
|
|||||||
<script>
|
<script>
|
||||||
function index() {
|
function index() {
|
||||||
return {
|
return {
|
||||||
items: [], version: {}, popups: {},
|
items: [], version: {},
|
||||||
fetch_version() {
|
fetch_version() {
|
||||||
fetch('/rest/version')
|
fetch('/rest/version')
|
||||||
.then(response => response.json())
|
.then(response => response.json())
|
||||||
@ -104,11 +79,9 @@
|
|||||||
})
|
})
|
||||||
},
|
},
|
||||||
show_popup(item) {
|
show_popup(item) {
|
||||||
// allegedly you can use the reference to pop the window to the front on subsequent
|
window.open(item.popup_url, item.id, "width={{ .Config.UI.PopupWidth }},height={{ .Config.UI.PopupHeight }}");
|
||||||
// clicks, but I can't seem to find a reliable way to do so.
|
},
|
||||||
this.popups[item.id] = window.open(item.popup_url, item.id, "width={{ .Config.UI.PopupWidth }},height={{ .Config.UI.PopupHeight }}");
|
}
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
{{ end }}
|
{{ end }}
|
||||||
@ -5,24 +5,9 @@
|
|||||||
<title>gropple</title>
|
<title>gropple</title>
|
||||||
<script src="/static/alpine.min.js" defer></script>
|
<script src="/static/alpine.min.js" defer></script>
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
<link rel="preconnect" href="https://rsms.me/">
|
<link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/pure-min.css" integrity="sha384-Uu6IeWbM+gzNVXJcM9XV3SohHtmWE+3VGi496jvgX1jyvDTXfdK+rfZc8C1Aehk5" crossorigin="anonymous">
|
||||||
<link rel="stylesheet" href="https://rsms.me/inter/inter.css">
|
<link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/grids-responsive-min.css">
|
||||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/pure-min.css" integrity="sha384-X38yfunGUhNzHpBaEBsWLO+A0HDYOQi8ufWDkZ0k9e0eXz/tH3II7uKZ9msv++Ls" crossorigin="anonymous">
|
|
||||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/grids-responsive-min.css">
|
|
||||||
<style>
|
<style>
|
||||||
|
|
||||||
:root {
|
|
||||||
font-family: Inter, sans-serif;
|
|
||||||
font-feature-settings: 'liga' 1, 'calt' 1; /* fix for Chrome */
|
|
||||||
}
|
|
||||||
@supports (font-variation-settings: normal) {
|
|
||||||
:root { font-family: InterVariable, sans-serif; }
|
|
||||||
}
|
|
||||||
|
|
||||||
.button-small {
|
|
||||||
font-size: 85%;
|
|
||||||
}
|
|
||||||
|
|
||||||
.pure-g > div {
|
.pure-g > div {
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
@ -34,9 +19,6 @@
|
|||||||
height: 100px;
|
height: 100px;
|
||||||
overflow:auto;
|
overflow:auto;
|
||||||
}
|
}
|
||||||
.filelist {
|
|
||||||
font-size: 60%;
|
|
||||||
}
|
|
||||||
footer {
|
footer {
|
||||||
padding-top: 50px;
|
padding-top: 50px;
|
||||||
font-size: 30%;
|
font-size: 30%;
|
||||||
@ -51,9 +33,6 @@
|
|||||||
.state-downloading {
|
.state-downloading {
|
||||||
color: blue;
|
color: blue;
|
||||||
}
|
}
|
||||||
.state-moved {
|
|
||||||
color: green;
|
|
||||||
}
|
|
||||||
.state-complete {
|
.state-complete {
|
||||||
color: green;
|
color: green;
|
||||||
}
|
}
|
||||||
@ -63,6 +42,9 @@
|
|||||||
.gropple-config input.input-long {
|
.gropple-config input.input-long {
|
||||||
width: 27em;
|
width: 27em;
|
||||||
}
|
}
|
||||||
|
.gropple-config button {
|
||||||
|
border-radius: 12px;
|
||||||
|
}
|
||||||
.gropple-config button.button-del {
|
.gropple-config button.button-del {
|
||||||
background: rgb(202, 60, 60);
|
background: rgb(202, 60, 60);
|
||||||
}
|
}
|
||||||
@ -75,7 +57,7 @@
|
|||||||
}
|
}
|
||||||
.error {
|
.error {
|
||||||
color: red;
|
color: red;
|
||||||
font-size: 120%;
|
font-size: 150%;
|
||||||
}
|
}
|
||||||
.success {
|
.success {
|
||||||
color: green;
|
color: green;
|
||||||
@ -89,8 +71,7 @@
|
|||||||
|
|
||||||
{{ template "content" . }}
|
{{ template "content" . }}
|
||||||
<footer>
|
<footer>
|
||||||
Homepage: <a href="https://github.com/tardisx/gropple">https://github.com/tardisx/gropple</a><br>
|
Homepage: <a href="https://github.com/tardisx/gropple">https://github.com/tardisx/gropple</a>
|
||||||
Version: {{ .Version.CurrentVersion }}
|
|
||||||
</footer>
|
</footer>
|
||||||
</body>
|
</body>
|
||||||
{{ template "js" . }}
|
{{ template "js" . }}
|
||||||
@ -7,11 +7,9 @@
|
|||||||
<li class="pure-menu-item">
|
<li class="pure-menu-item">
|
||||||
<a href="/config" class="pure-menu-link">Config</a>
|
<a href="/config" class="pure-menu-link">Config</a>
|
||||||
</li>
|
</li>
|
||||||
<li class="pure-menu-item">
|
|
||||||
<a href="/bulk" class="pure-menu-link">Bulk</a>
|
|
||||||
</li>
|
|
||||||
<li class="pure-menu-item">
|
<li class="pure-menu-item">
|
||||||
<a href="https://github.com/tardisx/gropple" class="pure-menu-link">Github</a>
|
<a href="https://github.com/tardisx/gropple" class="pure-menu-link">Github</a>
|
||||||
</li>
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@ -2,33 +2,31 @@
|
|||||||
<div id="layout" class="pure-g pure-u-1" x-data="popup()" x-init="fetch_data()">
|
<div id="layout" class="pure-g pure-u-1" x-data="popup()" x-init="fetch_data()">
|
||||||
<h2>Download started</h2>
|
<h2>Download started</h2>
|
||||||
<p>Fetching <tt>{{ .dl.Url }}</tt></p>
|
<p>Fetching <tt>{{ .dl.Url }}</tt></p>
|
||||||
<form class="pure-form">
|
|
||||||
<table class="pure-table" >
|
<table class="pure-table" >
|
||||||
<tr>
|
<tr>
|
||||||
<th>profile</th>
|
<th>profile</th>
|
||||||
<td>{{ .dl.DownloadProfile.Name }}</td>
|
|
||||||
</tr>
|
|
||||||
<tr><th>current filename</th><td x-text="filename"></td></tr>
|
|
||||||
<tr>
|
|
||||||
<th>option</th>
|
|
||||||
<td>
|
<td>
|
||||||
{{ if .dl.DownloadOption }} {{ .dl.DownloadOption.Name }} {{ else }} n/a {{ end }}
|
<select x-bind:disabled="profile_chosen" x-on:change="update_profile()" class="pure-input-1-2" x-model="profile_chosen">
|
||||||
|
<option value="">choose a profile to start</option>
|
||||||
|
{{ range $i := .config.DownloadProfiles }}
|
||||||
|
<option>{{ $i.Name }}</option>
|
||||||
|
{{ end }}
|
||||||
|
</select>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
<tr><th>current filename</th><td x-text="filename"></td></tr>
|
||||||
<tr><th>state</th><td x-text="state"></td></tr>
|
<tr><th>state</th><td x-text="state"></td></tr>
|
||||||
<tr x-show="playlist_total > 0"><th>playlist progress</th><td x-text="playlist_current + '/' + playlist_total"></td></tr>
|
|
||||||
<tr><th>progress</th><td x-text="percent"></td></tr>
|
<tr><th>progress</th><td x-text="percent"></td></tr>
|
||||||
<tr><th>ETA</th><td x-text="eta"></td></tr>
|
<tr><th>ETA</th><td x-text="eta"></td></tr>
|
||||||
|
|
||||||
</table>
|
</table>
|
||||||
<p>You can close this window and your download will continue. Check the <a href="/" target="_gropple_status">Status page</a> to see all downloads in progress.</p>
|
<p>You can close this window and your download will continue. Check the <a href="/" target="_gropple_status">Status page</a> to see all downloads in progress.</p>
|
||||||
{{ if .canStop }}
|
{{ if .canStop }}
|
||||||
<button x-show="state=='Downloading'" class="button-small pure-button" @click="stop()">stop</button>
|
<button x-show="state=='downloading'" class="pure-button" @click="stop()">stop</button>
|
||||||
{{ end }}
|
{{ end }}
|
||||||
</form>
|
|
||||||
<div>
|
<div>
|
||||||
<h4>Logs</h4>
|
<h4>Logs</h4>
|
||||||
<pre x-text="log" style="height: auto;">
|
<pre x-text="log">
|
||||||
</pre>
|
</pre>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@ -39,7 +37,23 @@
|
|||||||
history.replaceState(null, '', ['/fetch/{{ .dl.Id }}'])
|
history.replaceState(null, '', ['/fetch/{{ .dl.Id }}'])
|
||||||
return {
|
return {
|
||||||
eta: '', percent: 0.0, state: '??', filename: '', finished: false, log :'',
|
eta: '', percent: 0.0, state: '??', filename: '', finished: false, log :'',
|
||||||
playlist_current: 0, playlist_total: 0,
|
profile_chosen: null,
|
||||||
|
watch_profile() {
|
||||||
|
this.$watch('profile_chosen', value => this.profile_chosen(value))
|
||||||
|
},
|
||||||
|
update_profile(name) {
|
||||||
|
console.log('you chose name', this.profile_chosen);
|
||||||
|
let op = {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({action: 'start', profile: this.profile_chosen}),
|
||||||
|
headers: { 'Content-Type': 'application/json' }
|
||||||
|
};
|
||||||
|
fetch('/rest/fetch/{{ .dl.Id }}', op)
|
||||||
|
.then(response => response.json())
|
||||||
|
.then(info => {
|
||||||
|
console.log(info)
|
||||||
|
})
|
||||||
|
},
|
||||||
stop() {
|
stop() {
|
||||||
let op = {
|
let op = {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
@ -59,8 +73,9 @@
|
|||||||
this.eta = info.eta;
|
this.eta = info.eta;
|
||||||
this.percent = info.percent + "%";
|
this.percent = info.percent + "%";
|
||||||
this.state = info.state;
|
this.state = info.state;
|
||||||
this.playlist_current = info.playlist_current;
|
if (this.state != 'choose profile') {
|
||||||
this.playlist_total = info.playlist_total;
|
this.profile_chosen = true;
|
||||||
|
}
|
||||||
this.finished = info.finished;
|
this.finished = info.finished;
|
||||||
if (info.files && info.files.length > 0) {
|
if (info.files && info.files.length > 0) {
|
||||||
this.filename = info.files[info.files.length - 1];
|
this.filename = info.files[info.files.length - 1];
|
||||||
544
web/web.go
544
web/web.go
@ -1,544 +0,0 @@
|
|||||||
package web
|
|
||||||
|
|
||||||
import (
|
|
||||||
"embed"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"html/template"
|
|
||||||
"io"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/gorilla/mux"
|
|
||||||
"github.com/tardisx/gropple/config"
|
|
||||||
"github.com/tardisx/gropple/download"
|
|
||||||
"github.com/tardisx/gropple/version"
|
|
||||||
)
|
|
||||||
|
|
||||||
type successResponse struct {
|
|
||||||
Success bool `json:"success"`
|
|
||||||
Message string `json:"message"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type queuedResponse struct {
|
|
||||||
Success bool `json:"success"`
|
|
||||||
Location string `json:"location"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type errorResponse struct {
|
|
||||||
Success bool `json:"success"`
|
|
||||||
Error string `json:"error"`
|
|
||||||
}
|
|
||||||
|
|
||||||
//go:embed data/**
|
|
||||||
var webFS embed.FS
|
|
||||||
|
|
||||||
func CreateRoutes(cs *config.ConfigService, dm *download.Manager, vm *version.Manager) *mux.Router {
|
|
||||||
r := mux.NewRouter()
|
|
||||||
|
|
||||||
// main index page
|
|
||||||
r.HandleFunc("/", homeHandler(cs, vm, dm))
|
|
||||||
// update info on the status page
|
|
||||||
r.HandleFunc("/rest/fetch", fetchInfoRESTHandler(dm))
|
|
||||||
|
|
||||||
// return static files
|
|
||||||
r.HandleFunc("/static/{filename}", staticHandler())
|
|
||||||
|
|
||||||
// return the config page
|
|
||||||
r.HandleFunc("/config", configHandler())
|
|
||||||
// handle config fetches/updates
|
|
||||||
r.HandleFunc("/rest/config", configRESTHandler(cs))
|
|
||||||
|
|
||||||
// create or present a download in the popup
|
|
||||||
r.HandleFunc("/fetch", fetchHandler(cs, vm, dm))
|
|
||||||
r.HandleFunc("/fetch/{id}", fetchHandler(cs, vm, dm))
|
|
||||||
|
|
||||||
// handle the bulk uploader
|
|
||||||
r.HandleFunc("/bulk", bulkHandler(cs, vm, dm))
|
|
||||||
|
|
||||||
// get/update info on a download
|
|
||||||
r.HandleFunc("/rest/fetch/{id}", fetchInfoOneRESTHandler(cs, dm))
|
|
||||||
|
|
||||||
// version information
|
|
||||||
r.HandleFunc("/rest/version", versionRESTHandler(vm))
|
|
||||||
|
|
||||||
http.Handle("/", r)
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
// versionRESTHandler returns the version information, if we have up-to-date info from github
|
|
||||||
func versionRESTHandler(vm *version.Manager) func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if vm.GetInfo().GithubVersionFetched {
|
|
||||||
b, _ := json.Marshal(vm.GetInfo())
|
|
||||||
_, err := w.Write(b)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("could not write to client: %s", err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
w.WriteHeader(400)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// homeHandler returns the main index page
|
|
||||||
func homeHandler(cs *config.ConfigService, vm *version.Manager, dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
|
|
||||||
bookmarkletURL := fmt.Sprintf("javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('%s/fetch?url=',window.location,'yourform','width=%d,height=%d'));", cs.Config.Server.Address, cs.Config.UI.PopupWidth, cs.Config.UI.PopupHeight)
|
|
||||||
|
|
||||||
t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/menu.tmpl", "data/templates/index.tmpl")
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
type Info struct {
|
|
||||||
Manager *download.Manager
|
|
||||||
BookmarkletURL template.URL
|
|
||||||
Config *config.Config
|
|
||||||
Version version.Info
|
|
||||||
}
|
|
||||||
|
|
||||||
info := Info{
|
|
||||||
Manager: dm,
|
|
||||||
BookmarkletURL: template.URL(bookmarkletURL),
|
|
||||||
Config: cs.Config,
|
|
||||||
Version: vm.GetInfo(),
|
|
||||||
}
|
|
||||||
|
|
||||||
dm.Lock.Lock()
|
|
||||||
defer dm.Lock.Unlock()
|
|
||||||
err = t.ExecuteTemplate(w, "layout", info)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// staticHandler handles requests for static files
|
|
||||||
func staticHandler() func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
vars := mux.Vars(r)
|
|
||||||
|
|
||||||
filename := vars["filename"]
|
|
||||||
if strings.Index(filename, ".js") == len(filename)-3 {
|
|
||||||
f, err := webFS.Open("data/js/" + filename)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error accessing %s - %v", filename, err)
|
|
||||||
w.WriteHeader(http.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
_, err = io.Copy(w, f)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("could not write to client: %s", err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusNotFound)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// configHandler returns the configuration page
|
|
||||||
func configHandler() func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
|
|
||||||
t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/menu.tmpl", "data/templates/config.tmpl")
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
err = t.ExecuteTemplate(w, "layout", nil)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// configRESTHandler handles both reading and writing of the configuration
|
|
||||||
func configRESTHandler(cs *config.ConfigService) func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
|
|
||||||
if r.Method == "POST" {
|
|
||||||
log.Printf("Updating config")
|
|
||||||
b, err := io.ReadAll(r.Body)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
err = cs.Config.UpdateFromJSON(b)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
errorRes := errorResponse{Success: false, Error: err.Error()}
|
|
||||||
errorResB, _ := json.Marshal(errorRes)
|
|
||||||
w.WriteHeader(400)
|
|
||||||
_, err = w.Write(errorResB)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("could not write to client: %s", err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
cs.WriteConfig()
|
|
||||||
}
|
|
||||||
b, _ := json.Marshal(cs.Config)
|
|
||||||
_, err := w.Write(b)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("could not write config to client: %s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fetchInfoOneRESTHandler(cs *config.ConfigService, dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
vars := mux.Vars(r)
|
|
||||||
idString := vars["id"]
|
|
||||||
if idString != "" {
|
|
||||||
id, err := strconv.Atoi(idString)
|
|
||||||
if err != nil {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
thisDownload, err := dm.GetDlById(id)
|
|
||||||
if err != nil {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if thisDownload == nil {
|
|
||||||
panic("should not happen")
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Method == "POST" {
|
|
||||||
|
|
||||||
type updateRequest struct {
|
|
||||||
Action string `json:"action"`
|
|
||||||
}
|
|
||||||
|
|
||||||
thisReq := updateRequest{}
|
|
||||||
|
|
||||||
b, err := io.ReadAll(r.Body)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
err = json.Unmarshal(b, &thisReq)
|
|
||||||
if err != nil {
|
|
||||||
errorRes := errorResponse{Success: false, Error: err.Error()}
|
|
||||||
errorResB, _ := json.Marshal(errorRes)
|
|
||||||
w.WriteHeader(400)
|
|
||||||
_, err = w.Write(errorResB)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("could not write to client: %s", err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if thisReq.Action == "stop" {
|
|
||||||
|
|
||||||
thisDownload.Stop()
|
|
||||||
succRes := successResponse{Success: true, Message: "download stopped"}
|
|
||||||
succResB, _ := json.Marshal(succRes)
|
|
||||||
_, err = w.Write(succResB)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("could not write to client: %s", err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// just a get, return the object
|
|
||||||
thisDownload.Lock.Lock()
|
|
||||||
defer thisDownload.Lock.Unlock()
|
|
||||||
|
|
||||||
b, _ := json.Marshal(thisDownload)
|
|
||||||
|
|
||||||
_, err = w.Write(b)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("could not write to client: %s", err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
} else {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func fetchInfoRESTHandler(dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
|
|
||||||
b, err := dm.DownloadsAsJSON()
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
_, err = w.Write(b)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("could not write to client: %s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchHandler shows the popup, either the initial form (for create) or the form when in
|
|
||||||
// progress (to be updated by REST). It also handles the form POST for creating a new download.
|
|
||||||
func fetchHandler(cs *config.ConfigService, vm *version.Manager, dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
log.Printf("fetchHandler")
|
|
||||||
|
|
||||||
method := r.Method
|
|
||||||
|
|
||||||
// if they refreshed the popup, just load the existing object, don't
|
|
||||||
// create a new one
|
|
||||||
vars := mux.Vars(r)
|
|
||||||
idString := vars["id"]
|
|
||||||
|
|
||||||
idInt, idOK := strconv.ParseInt(idString, 10, 32)
|
|
||||||
|
|
||||||
if method == "GET" && idOK == nil && idInt > 0 {
|
|
||||||
// existing, load it up
|
|
||||||
log.Printf("loading popup for id %d", idInt)
|
|
||||||
dl, err := dm.GetDlById(int(idInt))
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("not found")
|
|
||||||
w.WriteHeader(404)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/popup.tmpl")
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
templateData := map[string]interface{}{"dl": dl, "config": cs.Config, "canStop": download.CanStopDownload, "Version": vm.GetInfo()}
|
|
||||||
|
|
||||||
err = t.ExecuteTemplate(w, "layout", templateData)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
return
|
|
||||||
} else if method == "POST" {
|
|
||||||
// creating a new one
|
|
||||||
type reqType struct {
|
|
||||||
URL string `json:"url"`
|
|
||||||
ProfileChosen string `json:"profile"`
|
|
||||||
DownloadOptionChosen string `json:"download_option"`
|
|
||||||
}
|
|
||||||
|
|
||||||
req := reqType{}
|
|
||||||
err := json.NewDecoder(r.Body).Decode(&req)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error decoding body of request: %s", err)
|
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("popup POST request: %#v", req)
|
|
||||||
|
|
||||||
if req.URL == "" {
|
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
|
||||||
_ = json.NewEncoder(w).Encode(errorResponse{
|
|
||||||
Success: false,
|
|
||||||
Error: "No URL supplied",
|
|
||||||
})
|
|
||||||
|
|
||||||
return
|
|
||||||
} else {
|
|
||||||
if req.ProfileChosen == "" {
|
|
||||||
|
|
||||||
w.WriteHeader(400)
|
|
||||||
_ = json.NewEncoder(w).Encode(errorResponse{
|
|
||||||
Success: false,
|
|
||||||
Error: "you must choose a profile",
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
profile := cs.Config.ProfileCalled(req.ProfileChosen)
|
|
||||||
if profile == nil {
|
|
||||||
w.WriteHeader(400)
|
|
||||||
_ = json.NewEncoder(w).Encode(errorResponse{
|
|
||||||
Success: false,
|
|
||||||
Error: fmt.Sprintf("no such profile: '%s'", req.ProfileChosen),
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
option := cs.Config.DownloadOptionCalled(req.DownloadOptionChosen)
|
|
||||||
|
|
||||||
// create the new download
|
|
||||||
newDL := download.NewDownload(req.URL, cs.Config)
|
|
||||||
id := newDL.Id
|
|
||||||
newDL.DownloadOption = option
|
|
||||||
newDL.DownloadProfile = *profile
|
|
||||||
dm.AddDownload(newDL)
|
|
||||||
dm.Queue(newDL)
|
|
||||||
|
|
||||||
w.WriteHeader(200)
|
|
||||||
_ = json.NewEncoder(w).Encode(queuedResponse{
|
|
||||||
Success: true,
|
|
||||||
Location: fmt.Sprintf("/fetch/%d", id),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// a GET, show the popup so they can start the download
|
|
||||||
log.Print("loading popup for a new download")
|
|
||||||
query := r.URL.Query()
|
|
||||||
url, present := query["url"]
|
|
||||||
|
|
||||||
if !present {
|
|
||||||
w.WriteHeader(400)
|
|
||||||
_, _ = fmt.Fprint(w, "No url supplied")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/popup_create.tmpl")
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
templateData := map[string]interface{}{"config": cs.Config, "url": url[0], "Version": vm.GetInfo()}
|
|
||||||
|
|
||||||
err = t.ExecuteTemplate(w, "layout", templateData)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func bulkHandler(cs *config.ConfigService, vm *version.Manager, dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
log.Printf("bulkHandler")
|
|
||||||
|
|
||||||
method := r.Method
|
|
||||||
switch method {
|
|
||||||
case "GET":
|
|
||||||
|
|
||||||
t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/menu.tmpl", "data/templates/bulk.tmpl")
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
templateData := map[string]interface{}{"config": cs.Config, "Version": vm.GetInfo()}
|
|
||||||
|
|
||||||
err = t.ExecuteTemplate(w, "layout", templateData)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
case "POST":
|
|
||||||
type reqBulkType struct {
|
|
||||||
URLs string `json:"urls"`
|
|
||||||
ProfileChosen string `json:"profile"`
|
|
||||||
DownloadOptionChosen string `json:"download_option"`
|
|
||||||
}
|
|
||||||
|
|
||||||
req := reqBulkType{}
|
|
||||||
err := json.NewDecoder(r.Body).Decode(&req)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error decoding request body: %s", err)
|
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
|
||||||
_, _ = w.Write([]byte(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("bulk POST request: %#v", req)
|
|
||||||
|
|
||||||
if req.URLs == "" {
|
|
||||||
w.WriteHeader(400)
|
|
||||||
_ = json.NewEncoder(w).Encode(errorResponse{
|
|
||||||
Success: false,
|
|
||||||
Error: "No URLs supplied",
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if req.ProfileChosen == "" {
|
|
||||||
|
|
||||||
w.WriteHeader(400)
|
|
||||||
_ = json.NewEncoder(w).Encode(errorResponse{
|
|
||||||
Success: false,
|
|
||||||
Error: "you must choose a profile",
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
profile := cs.Config.ProfileCalled(req.ProfileChosen)
|
|
||||||
if profile == nil {
|
|
||||||
w.WriteHeader(400)
|
|
||||||
_ = json.NewEncoder(w).Encode(errorResponse{
|
|
||||||
Success: false,
|
|
||||||
Error: fmt.Sprintf("no such profile: '%s'", req.ProfileChosen),
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
option := cs.Config.DownloadOptionCalled(req.DownloadOptionChosen)
|
|
||||||
|
|
||||||
// create the new downloads
|
|
||||||
urls := strings.Split(req.URLs, "\n")
|
|
||||||
count := 0
|
|
||||||
for _, thisURL := range urls {
|
|
||||||
thisURL = strings.TrimSpace(thisURL)
|
|
||||||
if thisURL != "" {
|
|
||||||
newDL := download.NewDownload(thisURL, cs.Config)
|
|
||||||
newDL.DownloadOption = option
|
|
||||||
newDL.DownloadProfile = *profile
|
|
||||||
dm.AddDownload(newDL)
|
|
||||||
dm.Queue(newDL)
|
|
||||||
log.Printf("queued %s", thisURL)
|
|
||||||
count++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
w.WriteHeader(200)
|
|
||||||
_ = json.NewEncoder(w).Encode(successResponse{
|
|
||||||
Success: true,
|
|
||||||
Message: fmt.Sprintf("queued %d downloads", count),
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
x
Reference in New Issue
Block a user