yt-db-completer/main.go

package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strings"
)

const (
	All        Category = "all"
	Shorts              = "shorts"
	Video               = "video"
	Livestream          = "livestream"
	Membership          = "membership"
)

type Category string

type Metadata struct {
	Id            string `json:"id"`
	Title         string `json:"title"`
	Date          string `json:"upload_date"`
	PlaylistTitle string `json:"playlist_title"`
}

func containsStr(s []string, id string) bool {
	for _, x := range s {
		if x == id {
			return true
		}
	}

	return false
}

func containsMeta(s []Metadata, id string) bool {
	for _, x := range s {
		if x.Id == id {
			return true
		}
	}

	return false
}

func (c *Category) String() string {
	return string(*c)
}

func (c *Category) Set(value string) error {
	switch strings.ToLower(value) {
	case "all":
		*c = All
	case "shorts":
		*c = Shorts
	case "video":
		*c = Video
	case "livestream":
		*c = Livestream
	case "membership":
		*c = Membership
	default:
		panic(fmt.Sprintf("Invalid category: %s", value))
	}
	return nil
}

func CategoryFlag(name string, value Category, usage string) *Category {
	c := value
	flag.Var(&c, name, usage)
	return &c
}

func getChannelID(handle string) string {
	resp, err := http.Get(fmt.Sprintf("https://www.youtube.com/%s", handle))
	if err != nil {
		log.Fatal(err)
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	re := regexp.MustCompile(`<meta[^>]*property=["']og:url["'][^>]*content=["']([^"']+)["']`)

	results := re.FindAllStringSubmatch(string(body), -1)
	parsedUrl := results[0][1]

	splits := strings.Split(parsedUrl, "/")
	channelID := splits[len(splits)-1]

	return channelID
}

func getMembersPlaylistID(channelID string) string {
	return fmt.Sprintf("UUMO%s", channelID[2:])
}

func main() {
	cat := CategoryFlag("category", All, "Category of videos to check against.\nValid options are all, shorts, video, livestream, membership")
	dumpToFile := flag.Bool("dump-to-file", false, "The program will dump the missing IDs to a file that can be used with yt-dlp")

	flag.Parse()

	args := flag.Args()

	if len(args) < 1 {
		fmt.Println("Please provide a json file dumped from yt-dlp as the first argument")
		return
	}

	if len(args) < 2 {
		fmt.Println("Please provide a directory with folder names formatted like this \"[YYYYMMDD] [XXXXXXXXXXXX] ...\" where X is the video ID")
		return
	}

	existingIds := make([]string, 0, 128)
	missingMetadata := make([]Metadata, 0, 128)
	extraIds := make([]string, 0, 128)
	allMetadata := make([]Metadata, 0, 1024)

	filePath := args[0]
	dirPath := args[1]

	dir, dirErr := os.Open(dirPath)
	defer dir.Close()
	if dirErr != nil {
		fmt.Printf("Error while opening provided directory: %v\n", dirErr)
		return
	}

	file, fileErr := os.Open(filePath)
	defer file.Close()
	if fileErr != nil {
		fmt.Printf("Error while opening provided json file: %v\n", fileErr)
		return
	}

	stat, statErr := dir.Stat()
	if statErr != nil {
		fmt.Printf("Error while stating the provided directory: %v\n", statErr)
		return
	}
	if !stat.IsDir() {
		fmt.Println("Second argument is not a directory")
		return
	}

	scanner := bufio.NewScanner(file)

	for scanner.Scan() {
		metadataJson := scanner.Text()

		var metadata Metadata
		json.Unmarshal([]byte(metadataJson), &metadata)

		switch *cat {
		case All: // Do nothing
		case Shorts:
			if !strings.HasSuffix(metadata.PlaylistTitle, "Shorts") {
				continue
			}
		case Video:
			if !strings.HasSuffix(metadata.PlaylistTitle, "Videos") {
				continue
			}
		case Livestream:
			if !strings.HasSuffix(metadata.PlaylistTitle, "Live") {
				continue
			}
		case Membership:
			if !strings.HasSuffix(metadata.PlaylistTitle, "Members-only videos") {
				continue
			}
		}

		allMetadata = append(allMetadata, metadata)
	}

	sort.Slice(allMetadata, func(a, b int) bool {
		return allMetadata[a].Date < allMetadata[b].Date
	})

	entries, dirErr := dir.ReadDir(0)
	if dirErr != nil {
		fmt.Printf("Error while reading directory contents: %v\n", dirErr)
	}

	for _, entry := range entries {
		if !entry.IsDir() {
			continue
		}

		// TODO: make sure all the directories are formatted properly by looking at the id and making sure
		// it looks like a youtube video id.
		// Warn the user if a directory doesn't conform to the id format.

		name := entry.Name()
		fields := strings.Fields(name)
		id := strings.Trim(strings.Trim(fields[1], "["), "]")
		existingIds = append(existingIds, id)
	}

	for _, meta := range allMetadata {
		if !containsStr(existingIds, meta.Id) {
			missingMetadata = append(missingMetadata, meta)
		}
	}

	for _, id := range existingIds {
		if !containsMeta(allMetadata, id) {
			extraIds = append(extraIds, id)
		}
	}

	if len(missingMetadata) == 0 {
		fmt.Printf("Found %d IDs. No missing IDs. Congrats\n", len(existingIds))
	} else {
		fmt.Printf("Found %d IDs. Still missing %d IDs. For a total of %d IDs\n", len(existingIds), len(allMetadata)-(len(existingIds)-len(extraIds)), len(allMetadata))

		fmt.Println("Missing Videos are:")
		fmt.Println(missingMetadata)
	}

	if len(extraIds) != 0 {
		fmt.Println("Found extra existing videos:")
		fmt.Println(extraIds)
	}

	if *dumpToFile {
		file, fileErr := os.Create("./missing.txt")
		defer file.Close()
		if fileErr != nil {
			fmt.Printf("Error while opening dump file: %v\n", fileErr)
			return
		}

		for _, meta := range missingMetadata {
			file.WriteString(meta.Id + "\n")
		}
	}
}