Sync from /srv/compose/unified-media-manager
This commit is contained in:
Christopher Mayor
427
internal/service/import.go
Normal file
427
internal/service/import.go
Normal file
@@ -0,0 +1,427 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/TopherMayor/unified-media-manager/internal/db"
|
||||
"github.com/TopherMayor/unified-media-manager/internal/download"
|
||||
)
|
||||
|
||||
type ImportResult struct {
|
||||
MediaID int64 `json:"media_id"`
|
||||
MediaType string `json:"media_type"`
|
||||
SourcePath string `json:"source_path"`
|
||||
DestPath string `json:"dest_path"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
Quality string `json:"quality"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
type ImportReport struct {
|
||||
Imported int `json:"imported"`
|
||||
Skipped int `json:"skipped"`
|
||||
Errors int `json:"errors"`
|
||||
Results []ImportResult `json:"results"`
|
||||
}
|
||||
|
||||
type ImportService struct {
|
||||
db *db.DB
|
||||
downloadClientSvc *DownloadClientService
|
||||
namingSvc *NamingService
|
||||
matcherSvc *MatcherService
|
||||
mediaSvc *MediaService
|
||||
parser *ReleaseParser
|
||||
downloadDir string
|
||||
subtitleSvc *SubtitleService
|
||||
activitySvc *ActivityService
|
||||
}
|
||||
|
||||
func NewImportService(database *db.DB, dcSvc *DownloadClientService, nSvc *NamingService, mSvc *MatcherService, mediaSvc *MediaService, downloadDir string, subtitleSvc *SubtitleService, activitySvc *ActivityService) *ImportService {
|
||||
return &ImportService{
|
||||
db: database,
|
||||
downloadClientSvc: dcSvc,
|
||||
namingSvc: nSvc,
|
||||
matcherSvc: mSvc,
|
||||
mediaSvc: mediaSvc,
|
||||
parser: NewReleaseParser(),
|
||||
downloadDir: downloadDir,
|
||||
subtitleSvc: subtitleSvc,
|
||||
activitySvc: activitySvc,
|
||||
}
|
||||
}
|
||||
|
||||
var mediaExts = map[string]bool{
|
||||
".mkv": true,
|
||||
".mp4": true,
|
||||
".avi": true,
|
||||
".wmv": true,
|
||||
".flv": true,
|
||||
".webm": true,
|
||||
".mp3": true,
|
||||
".flac": true,
|
||||
".m4a": true,
|
||||
".m4b": true,
|
||||
".ogg": true,
|
||||
".opus": true,
|
||||
".epub": true,
|
||||
".pdf": true,
|
||||
".mobi": true,
|
||||
".azw3": true,
|
||||
}
|
||||
|
||||
func (s *ImportService) ProcessCompleted(ctx context.Context) (*ImportReport, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
report := &ImportReport{}
|
||||
|
||||
nzbClients, err := s.downloadClientSvc.GetAllEnabled(ctx, "nzb")
|
||||
if err != nil {
|
||||
slog.Error("failed to get nzb clients", "error", err)
|
||||
}
|
||||
torrentClients, err := s.downloadClientSvc.GetAllEnabled(ctx, "torrent")
|
||||
if err != nil {
|
||||
slog.Error("failed to get torrent clients", "error", err)
|
||||
}
|
||||
|
||||
allClients := append(nzbClients, torrentClients...)
|
||||
|
||||
for _, client := range allClients {
|
||||
completed, err := client.Client.GetCompleted(ctx)
|
||||
if err != nil {
|
||||
slog.Error("failed to get completed downloads", "error", err, "client", client.Config.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, dl := range completed {
|
||||
s.processDownload(ctx, dl, client, report)
|
||||
}
|
||||
}
|
||||
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (s *ImportService) processDownload(ctx context.Context, dl download.CompletedDownload, client DownloadClientWithInfo, report *ImportReport) {
|
||||
var exists bool
|
||||
err := s.db.Pool.QueryRow(ctx,
|
||||
"SELECT EXISTS(SELECT 1 FROM media_files WHERE original_path = $1 AND deleted_at IS NULL)",
|
||||
dl.OutputPath).Scan(&exists)
|
||||
if err != nil {
|
||||
slog.Error("failed to check existing import", "error", err, "path", dl.OutputPath)
|
||||
report.Errors++
|
||||
return
|
||||
}
|
||||
if exists {
|
||||
report.Skipped++
|
||||
return
|
||||
}
|
||||
|
||||
files, err := s.findMediaFiles(dl.Name)
|
||||
if err != nil {
|
||||
slog.Error("failed to find media files", "error", err, "download", dl.Name)
|
||||
report.Errors++
|
||||
return
|
||||
}
|
||||
|
||||
if len(files) == 0 {
|
||||
slog.Warn("no media files found for download", "download", dl.Name)
|
||||
report.Skipped++
|
||||
return
|
||||
}
|
||||
|
||||
for _, filePath := range files {
|
||||
s.processFile(ctx, filePath, dl, client, report)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ImportService) processFile(ctx context.Context, sourcePath string, dl download.CompletedDownload, client DownloadClientWithInfo, report *ImportReport) {
|
||||
fileCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
releaseInfo := s.parser.Parse(filepath.Base(sourcePath))
|
||||
|
||||
mediaType := "movie"
|
||||
if _, _, hasSE := parseSeasonEpisode(filepath.Base(sourcePath)); hasSE {
|
||||
mediaType = "series"
|
||||
}
|
||||
|
||||
match, err := s.matcherSvc.Match(fileCtx, dl.Name, mediaType)
|
||||
if err != nil {
|
||||
slog.Error("failed to match release to media", "error", err, "release", dl.Name)
|
||||
report.Errors++
|
||||
return
|
||||
}
|
||||
if match.Confidence == "none" {
|
||||
slog.Warn("no media match for release", "release", dl.Name, "path", sourcePath)
|
||||
report.Skipped++
|
||||
return
|
||||
}
|
||||
|
||||
result, err := s.importFile(fileCtx, sourcePath, match, releaseInfo, dl, client)
|
||||
if err != nil {
|
||||
slog.Error("failed to import file", "error", err, "source", sourcePath)
|
||||
report.Errors++
|
||||
return
|
||||
}
|
||||
|
||||
report.Imported++
|
||||
report.Results = append(report.Results, *result)
|
||||
}
|
||||
|
||||
func (s *ImportService) importFile(ctx context.Context, sourcePath string, match *MatchResult, releaseInfo ReleaseInfo, completed download.CompletedDownload, client DownloadClientWithInfo) (*ImportResult, error) {
|
||||
status := "importing"
|
||||
err := s.mediaSvc.Update(ctx, match.MediaID, match.MediaType, UpdateMediaRequest{
|
||||
Status: &status,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("update media status to importing: %w", err)
|
||||
}
|
||||
|
||||
qualityTier := s.parser.MatchQuality(releaseInfo)
|
||||
qualityJSON, _ := json.Marshal(qualityTier)
|
||||
|
||||
year := 0
|
||||
if match.Year != nil {
|
||||
year = *match.Year
|
||||
}
|
||||
season := 0
|
||||
if match.Season != nil {
|
||||
season = *match.Season
|
||||
}
|
||||
episode := 0
|
||||
if match.Episode != nil {
|
||||
episode = *match.Episode
|
||||
}
|
||||
|
||||
namingData := NamingData{
|
||||
Title: match.Title,
|
||||
Year: year,
|
||||
Season: season,
|
||||
Episode: episode,
|
||||
Quality: qualityTier.Name,
|
||||
Ext: ExtractExt(filepath.Base(sourcePath)),
|
||||
ReleaseGroup: releaseInfo.ReleaseGroup,
|
||||
Resolution: releaseInfo.Resolution,
|
||||
Source: releaseInfo.Source,
|
||||
Codec: releaseInfo.VideoCodec,
|
||||
}
|
||||
|
||||
relativePath, err := s.namingSvc.Render(ctx, match.MediaType, namingData)
|
||||
if err != nil {
|
||||
s.rollbackStatus(ctx, match.MediaID, match.MediaType, "failed")
|
||||
s.logImportError(match.MediaID, match.MediaType, fmt.Sprintf("Naming template failed for media %d: %v", match.MediaID, err))
|
||||
return nil, fmt.Errorf("render naming template: %w", err)
|
||||
}
|
||||
|
||||
targetPath := filepath.Join(match.RootFolder, relativePath)
|
||||
|
||||
if !strings.HasPrefix(filepath.Clean(targetPath), filepath.Clean(match.RootFolder)) {
|
||||
s.rollbackStatus(ctx, match.MediaID, match.MediaType, "failed")
|
||||
return nil, fmt.Errorf("path traversal detected: target path escapes root folder")
|
||||
}
|
||||
|
||||
targetDir := filepath.Dir(targetPath)
|
||||
if err := os.MkdirAll(targetDir, 0755); err != nil {
|
||||
s.rollbackStatus(ctx, match.MediaID, match.MediaType, "failed")
|
||||
return nil, fmt.Errorf("create target directory: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Link(sourcePath, targetPath); err != nil {
|
||||
s.rollbackStatus(ctx, match.MediaID, match.MediaType, "failed")
|
||||
s.logImportError(match.MediaID, match.MediaType, fmt.Sprintf("Hardlink failed for media %d: %v", match.MediaID, err))
|
||||
return nil, fmt.Errorf("hardlink file: %w", err)
|
||||
}
|
||||
|
||||
srcInfo, err := os.Stat(sourcePath)
|
||||
if err != nil {
|
||||
s.rollbackStatus(ctx, match.MediaID, match.MediaType, "failed")
|
||||
return nil, fmt.Errorf("stat source file: %w", err)
|
||||
}
|
||||
dstInfo, err := os.Stat(targetPath)
|
||||
if err != nil {
|
||||
s.rollbackStatus(ctx, match.MediaID, match.MediaType, "failed")
|
||||
return nil, fmt.Errorf("stat target file: %w", err)
|
||||
}
|
||||
if !os.SameFile(srcInfo, dstInfo) {
|
||||
s.rollbackStatus(ctx, match.MediaID, match.MediaType, "failed")
|
||||
return nil, fmt.Errorf("hardlink verification failed: files are not the same inode")
|
||||
}
|
||||
|
||||
fileSize := dstInfo.Size()
|
||||
|
||||
if s.subtitleSvc != nil {
|
||||
extractCtx, extractCancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
baseName := s.buildImportSubtitleBaseName(match, releaseInfo)
|
||||
extracted, err := s.subtitleSvc.ExtractSubtitles(extractCtx, targetPath, filepath.Dir(targetPath), baseName)
|
||||
if err != nil {
|
||||
slog.Error("failed to extract subtitles", "error", err, "path", targetPath)
|
||||
}
|
||||
if len(extracted) > 0 {
|
||||
slog.Info("extracted subtitles", "count", len(extracted), "media_id", match.MediaID)
|
||||
}
|
||||
extractCancel()
|
||||
}
|
||||
|
||||
_, err = s.db.Pool.Exec(ctx,
|
||||
`INSERT INTO media_files (media_id, media_type, path, original_path, file_name, file_size, quality, codec, resolution, source, is_hardlinked)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
|
||||
match.MediaID, match.MediaType, targetPath, sourcePath, filepath.Base(targetPath),
|
||||
fileSize, qualityJSON, ptrStr(releaseInfo.VideoCodec), ptrStr(releaseInfo.Resolution),
|
||||
ptrStr(releaseInfo.Source), true)
|
||||
if err != nil {
|
||||
s.rollbackStatus(ctx, match.MediaID, match.MediaType, "failed")
|
||||
return nil, fmt.Errorf("insert media file record: %w", err)
|
||||
}
|
||||
|
||||
availableStatus := "available"
|
||||
err = s.mediaSvc.Update(ctx, match.MediaID, match.MediaType, UpdateMediaRequest{
|
||||
Status: &availableStatus,
|
||||
CurrentQuality: qualityJSON,
|
||||
})
|
||||
if err != nil {
|
||||
slog.Error("failed to update media status to available", "error", err, "media_id", match.MediaID)
|
||||
}
|
||||
|
||||
if _, err := s.db.Pool.Exec(ctx, `UPDATE media SET has_files = true WHERE id = $1`, match.MediaID); err != nil {
|
||||
slog.Error("failed to update has_files", "error", err, "media_id", match.MediaID)
|
||||
}
|
||||
|
||||
// Log successful import activity
|
||||
if s.activitySvc != nil {
|
||||
s.activitySvc.LogAsync(LogEntry{
|
||||
EventType: "import",
|
||||
MediaID: &match.MediaID,
|
||||
MediaType: &match.MediaType,
|
||||
Title: fmt.Sprintf("Imported %s", filepath.Base(sourcePath)),
|
||||
Data: json.RawMessage(fmt.Sprintf(`{"source":"%s","dest":"%s","quality":"%s","size":%d}`,
|
||||
sourcePath, targetPath, qualityTier.Name, fileSize)),
|
||||
})
|
||||
}
|
||||
|
||||
_, err = s.db.Pool.Exec(ctx,
|
||||
`UPDATE download_queue SET status = 'imported', completed_at = NOW()
|
||||
WHERE media_id = $1 AND release_title = $2 AND status IN ('downloading', 'pending')`,
|
||||
match.MediaID, completed.Name)
|
||||
if err != nil {
|
||||
slog.Error("failed to update download queue", "error", err, "media_id", match.MediaID)
|
||||
}
|
||||
|
||||
if err := client.Client.Remove(ctx, completed.ID); err != nil {
|
||||
slog.Warn("failed to remove download client entry", "error", err, "id", completed.ID)
|
||||
}
|
||||
|
||||
return &ImportResult{
|
||||
MediaID: match.MediaID,
|
||||
MediaType: match.MediaType,
|
||||
SourcePath: sourcePath,
|
||||
DestPath: targetPath,
|
||||
FileSize: fileSize,
|
||||
Quality: qualityTier.Name,
|
||||
Status: "imported",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ImportService) rollbackStatus(ctx context.Context, mediaID int64, mediaType string, status string) {
|
||||
if err := s.mediaSvc.Update(ctx, mediaID, mediaType, UpdateMediaRequest{Status: &status}); err != nil {
|
||||
slog.Error("failed to rollback media status", "error", err, "media_id", mediaID)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ImportService) findMediaFiles(downloadName string) ([]string, error) {
|
||||
downloadPath := filepath.Join(s.downloadDir, downloadName)
|
||||
cleanBase := filepath.Clean(s.downloadDir)
|
||||
|
||||
info, err := os.Stat(downloadPath)
|
||||
if err != nil {
|
||||
entries, err := os.ReadDir(s.downloadDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read download directory: %w", err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
candidate := filepath.Join(s.downloadDir, entry.Name())
|
||||
if strings.Contains(strings.ToLower(entry.Name()), strings.ToLower(downloadName)) {
|
||||
if entry.IsDir() {
|
||||
return s.walkMediaDir(candidate, cleanBase)
|
||||
}
|
||||
if mediaExts[filepath.Ext(entry.Name())] {
|
||||
return []string{candidate}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(filepath.Clean(downloadPath), cleanBase) {
|
||||
return nil, fmt.Errorf("path traversal detected: download path escapes download dir")
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return s.walkMediaDir(downloadPath, cleanBase)
|
||||
}
|
||||
|
||||
if mediaExts[filepath.Ext(downloadPath)] {
|
||||
return []string{downloadPath}, nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *ImportService) walkMediaDir(dir string, cleanBase string) ([]string, error) {
|
||||
var files []string
|
||||
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !strings.HasPrefix(filepath.Clean(path), cleanBase) {
|
||||
return fmt.Errorf("path traversal detected: walked path escapes download dir")
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if mediaExts[filepath.Ext(path)] {
|
||||
files = append(files, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("walk download directory: %w", err)
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func (s *ImportService) buildImportSubtitleBaseName(match *MatchResult, info ReleaseInfo) string {
|
||||
parts := []string{sanitize(match.Title)}
|
||||
if match.Year != nil {
|
||||
parts = append(parts, fmt.Sprintf("%d", *match.Year))
|
||||
}
|
||||
if match.Season != nil && match.Episode != nil {
|
||||
parts = append(parts, fmt.Sprintf("S%02dE%02d", *match.Season, *match.Episode))
|
||||
}
|
||||
return strings.Join(parts, ".")
|
||||
}
|
||||
|
||||
func ptrStr(s string) *string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
return &s
|
||||
}
|
||||
|
||||
func (s *ImportService) logImportError(mediaID int64, mediaType string, msg string) {
|
||||
if s.activitySvc != nil {
|
||||
s.activitySvc.LogAsync(LogEntry{
|
||||
EventType: "error",
|
||||
Title: msg,
|
||||
MediaID: &mediaID,
|
||||
MediaType: &mediaType,
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user