xplshn
·
2025-08-13
generator.go
Go
1package main
2
3import (
4 "fmt"
5 "io"
6 "net/http"
7 "os"
8 "sort"
9 "strconv"
10 "strings"
11
12 "github.com/fxamacker/cbor/v2"
13 "github.com/goccy/go-json"
14 minify "github.com/tdewolff/minify/v2"
15 mjson "github.com/tdewolff/minify/v2/json"
16 "github.com/tiendc/go-deepcopy"
17)
18
19type repository struct {
20 URL string
21 Name string
22 Single bool
23}
24
25type PkgForgeItem struct {
26 Pkg string `json:"pkg"`
27 Name string `json:"pkg_name,omitempty"`
28 Family string `json:"pkg_family,omitempty"`
29 PkgId string `json:"pkg_id,omitempty"`
30 AppId string `json:"app_id,omitempty"`
31 PkgType string `json:"pkg_type,omitempty"`
32 Icon string `json:"icon,omitempty"`
33 Screenshots []string `json:"screenshots,omitempty"`
34 Description string `json:"description,omitempty"`
35 Homepage []string `json:"homepage,omitempty"`
36 Version string `json:"version,omitempty"`
37 DownloadURL string `json:"download_url,omitempty"`
38 Size string `json:"size,omitempty"`
39 Bsum string `json:"bsum,omitempty"`
40 Shasum string `json:"shasum,omitempty"`
41 BuildDate string `json:"build_date,omitempty"`
42 SrcURL []string `json:"src_url,omitempty"`
43 BuildScript string `json:"build_script,omitempty"`
44 BuildLog string `json:"build_log,omitempty"`
45 Category []string `json:"categories,omitempty"`
46 Snapshots []string `json:"snapshots,omitempty"`
47 Provides []string `json:"provides,omitempty"`
48 Notes []string `json:"note,omitempty"`
49 License []string `json:"license,omitempty"`
50 GhcrPkg string `json:"ghcr_pkg,omitempty"`
51 HfPkg string `json:"hf_pkg,omitempty"`
52 Rank string `json:"rank,omitempty"`
53}
54
55type snapshot struct {
56 Commit string `json:"commit,omitempty"`
57 Version string `json:"version,omitempty"`
58}
59
60type DbinItem struct {
61 Pkg string `json:"pkg,omitempty"`
62 Name string `json:"pkg_name,omitempty"`
63 PkgId string `json:"pkg_id,omitempty"`
64 AppStreamId string `json:"app_id,omitempty"`
65 Icon string `json:"icon,omitempty"`
66 Description string `json:"description,omitempty"`
67 LongDescription string `json:"description_long,omitempty"`
68 Screenshots []string `json:"screenshots,omitempty"`
69 Version string `json:"version,omitempty"`
70 DownloadURL string `json:"download_url,omitempty"`
71 Size string `json:"size,omitempty"`
72 Bsum string `json:"bsum,omitempty"`
73 Shasum string `json:"shasum,omitempty"`
74 BuildDate string `json:"build_date,omitempty"`
75 SrcURLs []string `json:"src_urls,omitempty"`
76 WebURLs []string `json:"web_urls,omitempty"`
77 BuildScript string `json:"build_script,omitempty"`
78 BuildLog string `json:"build_log,omitempty"`
79 Categories string `json:"categories,omitempty"`
80 Snapshots []snapshot `json:"snapshots,omitempty"`
81 Provides string `json:"provides,omitempty"`
82 License []string `json:"license,omitempty"`
83 Notes []string `json:"notes,omitempty"`
84 Appstream string `json:"appstream,omitempty"`
85 Rank uint `json:"rank,omitempty"`
86}
87
88type DbinMetadata map[string][]DbinItem
89
90type RepositoryHandler interface {
91 FetchMetadata(url string) ([]DbinItem, error)
92}
93
94type PkgForgeHandler struct{}
95
96func (PkgForgeHandler) FetchMetadata(url string) ([]DbinItem, error) {
97 return fetchAndConvertMetadata(url, downloadJSON, convertPkgForgeToDbinItem)
98}
99
100type DbinHandler struct{}
101
102func (DbinHandler) FetchMetadata(url string) ([]DbinItem, error) {
103 resp, err := http.Get(url)
104 if err != nil {
105 return nil, err
106 }
107 defer resp.Body.Close()
108
109 body, err := io.ReadAll(resp.Body)
110 if err != nil {
111 return nil, err
112 }
113
114 var metadata DbinMetadata
115 err = json.Unmarshal(body, &metadata)
116 if err != nil {
117 return nil, err
118 }
119
120 // Since the metadata is already in Dbin format, we just need to extract the items
121 for _, items := range metadata {
122 return items, nil
123 }
124
125 return nil, nil
126}
127
128func fetchAndConvertMetadata(url string, downloadFunc func(string) ([]PkgForgeItem, error), convertFunc func(PkgForgeItem, map[string]bool) (DbinItem, bool)) ([]DbinItem, error) {
129 items, err := downloadFunc(url)
130 if err != nil {
131 return nil, err
132 }
133
134 familyCount := make(map[string]int)
135 familyNames := make(map[string]string)
136 useFamilyFormat := make(map[string]bool)
137
138 for _, item := range items {
139 familyCount[item.Family]++
140 if familyNames[item.Family] == "" {
141 familyNames[item.Family] = item.Name
142 } else if familyNames[item.Family] != item.Name {
143 useFamilyFormat[item.Family] = true
144 }
145 }
146
147 var dbinItems []DbinItem
148 for _, item := range items {
149 dbinItem, include := convertFunc(item, useFamilyFormat)
150 if include {
151 dbinItems = append(dbinItems, dbinItem)
152 }
153 }
154
155 return dbinItems, nil
156}
157
158func convertPkgForgeToDbinItem(item PkgForgeItem, useFamilyFormat map[string]bool) (DbinItem, bool) {
159 // PkgTypes we discard, completely
160 if item.PkgType == "dynamic" {
161 return DbinItem{}, false
162 }
163
164 var categories, provides, downloadURL string
165
166 if len(item.Category) > 0 {
167 categories = strings.Join(item.Category, ",")
168 }
169
170 if len(item.Provides) > 0 {
171 provides = strings.Join(item.Provides, ",")
172 }
173
174 if item.GhcrPkg != "" {
175 downloadURL = "oci://" + item.GhcrPkg
176 } else if item.HfPkg != "" {
177 downloadURL = strings.Replace(item.HfPkg, "/tree/main", "/resolve/main", 1) + "/" + item.Pkg
178 }
179
180 rank, _ := strconv.Atoi(item.Rank)
181
182 // Parse snapshots
183 var snapshots []snapshot
184 for _, snapshotStr := range item.Snapshots {
185 parts := strings.Split(snapshotStr, "[")
186 commit := strings.TrimSpace(parts[0])
187 version := ""
188 if len(parts) > 1 {
189 version = strings.TrimSuffix(parts[1], "]")
190 }
191 snapshots = append(snapshots, snapshot{Commit: commit, Version: version})
192 }
193
194 // - Determine the package name format
195 // | - If all packages in a family have the same name (e.g., "bwrap" in the "bubblewrap" family),
196 // | the package name will be just the package name (e.g., "bwrap").
197 // | - If there are multiple packages with different names in a family, the format will be
198 // | "family/package_name" (e.g., "a-utils/ccat").
199 // - Applies to all occurrences
200 pkgName := item.Name
201 if useFamilyFormat[item.Family] {
202 pkgName = fmt.Sprintf("%s/%s", item.Family, item.Name)
203 }
204
205 if item.PkgType == "static" {
206 pkgName = strings.TrimSuffix(pkgName, ".static")
207 } else if item.PkgType == "archive" {
208 pkgName = strings.TrimSuffix(pkgName, ".archive")
209 } else if item.PkgType != "" {
210 pkgName = pkgName + "." + item.PkgType
211 }
212
213 return DbinItem{
214 Pkg: pkgName,
215 Name: item.Name,
216 PkgId: item.PkgId,
217 AppStreamId: item.AppId,
218 Icon: item.Icon,
219 Screenshots: item.Screenshots,
220 Description: item.Description,
221 Version: item.Version,
222 DownloadURL: downloadURL,
223 Size: item.Size,
224 Bsum: item.Bsum,
225 Shasum: item.Shasum,
226 BuildDate: item.BuildDate,
227 SrcURLs: item.SrcURL,
228 WebURLs: item.Homepage,
229 BuildScript: item.BuildScript,
230 BuildLog: item.BuildLog,
231 Categories: categories,
232 Snapshots: snapshots,
233 Provides: provides,
234 License: item.License,
235 Notes: item.Notes,
236 Rank: uint(rank),
237 }, true
238}
239
240func downloadJSON(url string) ([]PkgForgeItem, error) {
241 resp, err := http.Get(url)
242 if err != nil {
243 return nil, err
244 }
245 defer resp.Body.Close()
246
247 body, err := io.ReadAll(resp.Body)
248 if err != nil {
249 return nil, err
250 }
251
252 var items []PkgForgeItem
253 err = json.Unmarshal(body, &items)
254 if err != nil {
255 return nil, err
256 }
257
258 return items, nil
259}
260
261func reorderItems(str []map[string]string, metadata DbinMetadata) {
262 for _, replacements := range str {
263 for repo, items := range metadata {
264 // Replace str with str2
265 for oldStr, newStr := range replacements {
266 for i := range items {
267 items[i].PkgId = strings.ReplaceAll(items[i].PkgId, oldStr, newStr)
268 }
269 }
270
271 // Sort items alphabetically by BinId
272 sort.Slice(items, func(i, j int) bool {
273 return items[i].PkgId < items[j].PkgId
274 })
275
276 // Replace str2 back to str
277 for oldStr, newStr := range replacements {
278 for i := range items {
279 items[i].PkgId = strings.ReplaceAll(items[i].PkgId, newStr, oldStr)
280 }
281 }
282
283 metadata[repo] = items
284 }
285 }
286}
287
288func saveAll(filename string, metadata DbinMetadata) error {
289 if err := saveJSON(filename, metadata); err != nil {
290 return err
291 }
292 return saveCBOR(filename, metadata)
293}
294
295func saveMetadata(filename string, metadata DbinMetadata) error {
296 // Reorder items alphabetically but with priority exceptions, to ensure a higher level of quality.
297 // We basically do a search&replace, order alphabetically, and then do a search&replace again.
298 // I prioritize binaries with a smaller size, more hardware compat, and that are truly static.
299 reorderItems([]map[string]string{
300 {"musl": "0AAAMusl"}, // | Higher priority for Musl
301 {"ppkg": "0AABPpkg"}, // | Higher priority for ppkg
302 {"glibc": "ZZZXXXGlibc"}, // | Push glibc to the end
303 // - // | - Little Glenda says hi!
304 // - // | (\(\
305 {"musl-v3": "0AACMusl"}, // | ¸". ..
306 {"glibc-v3": "ZZZXXXXGlibc"}, // | ( . .)
307 // - // | | ° ¡
308 {"musl-v4": "0AADMusl"}, // | ¿ ;
309 {"glibc-v4": "ZZZXXXZGlibc"}, // | c?".UJ"
310 }, metadata)
311
312 if err := saveAll(filename, metadata); err != nil {
313 return err
314 }
315
316 // "web" version
317 var webMetadata DbinMetadata
318 _ = deepcopy.Copy(&webMetadata, &metadata)
319 for _, items := range webMetadata {
320 for i := range items {
321 items[i].Provides = ""
322 items[i].Shasum = ""
323 items[i].Bsum = ""
324 items[i].AppStreamId = ""
325 }
326 }
327 saveAll(filename+".web", webMetadata)
328 // "lite" version
329 for _, items := range metadata {
330 for i := range items {
331 items[i].Icon = ""
332 items[i].Provides = ""
333 items[i].Shasum = ""
334 items[i].AppStreamId = ""
335 items[i].Screenshots = []string{}
336 }
337 }
338 return saveAll(filename+".lite", metadata)
339}
340
341func saveCBOR(filename string, metadata DbinMetadata) error {
342 cborData, err := cbor.Marshal(metadata)
343 if err != nil {
344 return err
345 }
346 return os.WriteFile(filename+".cbor", cborData, 0644)
347}
348
349func saveJSON(filename string, metadata DbinMetadata) error {
350 jsonData, err := json.MarshalIndent(metadata, "", " ")
351 if err != nil {
352 return err
353 }
354 if err := os.WriteFile(filename+".json", jsonData, 0644); err != nil {
355 return err
356 }
357 // Minify JSON
358 m := minify.New()
359 m.AddFunc("application/json", mjson.Minify)
360 if jsonData, err = m.Bytes("application/json", jsonData); err != nil {
361 return err
362 } else if err := os.WriteFile(filename+".min.json", jsonData, 0644); err != nil {
363 return err
364 }
365 return nil
366}
367
368func main() {
369 realArchs := map[string]string{
370 "x86_64-Linux": "amd64_linux",
371 "aarch64-Linux": "arm64_linux",
372 "riscv64-Linux": "riscv64_linux",
373 }
374
375 repositories := []struct {
376 Repo repository
377 Handler RepositoryHandler
378 }{
379 {
380 Repo: repository{
381 Name: "bincache",
382 URL: "https://meta.pkgforge.dev/bincache/%s.json",
383 Single: true,
384 },
385 Handler: PkgForgeHandler{},
386 },
387 {
388 Repo: repository{
389 Name: "pkgcache",
390 URL: "https://meta.pkgforge.dev/pkgcache/%s.json",
391 Single: true,
392 },
393 Handler: PkgForgeHandler{},
394 },
395 //{
396 // Repo: repository{
397 // Name: "pkgforge-go",
398 // URL: "https://meta.pkgforge.dev/external/pkgforge-go/%s.json",
399 // Standalone: true,
400 // },
401 // Handler: PkgForgeHandler{},
402 //},
403 //{
404 // Repo: repository{
405 // Name: "pkgforge-cargo",
406 // URL: "https://meta.pkgforge.dev/external/pkgforge-cargo/%s.json",
407 // Standalone: true,
408 // },
409 // Handler: PkgForgeHandler{},
410 //},
411 //{
412 // Repo: repository{
413 // Name: "AM",
414 // URL: "https://meta.pkgforge.dev/external/am/%s.json",
415 // Standalone: true,
416 // },
417 // Handler: PkgForgeHandler{},
418 //},
419 //{
420 // Repo: repository{
421 // Name: "appimage-github-io",
422 // URL: "https://meta.pkgforge.dev/external/appimage.github.io/%s.json",
423 // Standalone: true,
424 // },
425 // Handler: PkgForgeHandler{},
426 //},
427 {
428 Repo: repository{
429 Name: "AppBundleHUB",
430 URL: "https://github.com/xplshn/AppBundleHUB/releases/download/latest_metadata/metadata_%s.json",
431 Single: true,
432 },
433 Handler: DbinHandler{},
434 },
435 //{
436 // Repo: repository{
437 // Name: "dbin",
438 // URL: "http://192.168.1.59/d/%s",
439 // Single: true,
440 // },
441 // Handler: DbinHandler{},
442 //},
443 }
444
445 for arch, outputArch := range realArchs {
446 dbinMetadata := make(DbinMetadata)
447
448 for _, repo := range repositories {
449 url := repo.Repo.URL
450 if strings.Contains(url, "%s") {
451 url = fmt.Sprintf(url, arch)
452 }
453
454 items, err := repo.Handler.FetchMetadata(url)
455 if err != nil {
456 fmt.Printf("Error downloading %s metadata from %s: %v\n", repo.Repo.Name, url, err)
457 continue
458 }
459
460 // Filter items from "pkgcache" repository that do not contain "[PORTABLE]" in their Notes
461 if repo.Repo.Name == "pkgcache" {
462 var filteredItems []DbinItem
463 for _, item := range items {
464 hasPortableNote := false
465 for _, note := range item.Notes {
466 if strings.Contains(note, "[PORTABLE]") {
467 hasPortableNote = true
468 break
469 }
470 }
471 if hasPortableNote {
472 filteredItems = append(filteredItems, item)
473 }
474 }
475 items = filteredItems
476 }
477
478 dbinMetadata[repo.Repo.Name] = append(dbinMetadata[repo.Repo.Name], items...)
479
480 if repo.Repo.Single {
481 singleMetadata := make(DbinMetadata)
482 singleMetadata[repo.Repo.Name] = items
483 singleOutputFile := fmt.Sprintf("%s_%s", repo.Repo.Name, outputArch)
484
485 if err := saveMetadata(singleOutputFile, singleMetadata); err != nil {
486 fmt.Printf("Error saving single metadata to %s: %v\n", singleOutputFile, err)
487 continue
488 }
489 fmt.Printf("Successfully saved single metadata to %s\n", singleOutputFile)
490 }
491 }
492
493 outputFile := fmt.Sprintf("%s", outputArch)
494 if err := saveMetadata(outputFile, dbinMetadata); err != nil {
495 fmt.Printf("Error saving metadata to %s: %v\n", outputFile, err)
496 continue
497 }
498
499 fmt.Printf("Successfully processed and saved combined metadata to %s\n", outputFile)
500 }
501}
502
503func t[T any](cond bool, vtrue, vfalse T) T {
504 if cond {
505 return vtrue
506 }
507 return vfalse
508}
509
510/* The following is a _favor_ I'm doing to ivan-hc and everyone that contributes
511 * And actively endorses or uses AM
512 * They are a tremendous help to the Portable Linux Apps community!
513const pipeRepl = "ǀ" // Replacement for `|` to avoid breaking the MD table
514func replacePipeFields(pkg *DbinItem) {
515 pkg.Name = strings.ReplaceAll(pkg.Name, "|", pipeRepl)
516 pkg.Description = strings.ReplaceAll(pkg.Description, "|", pipeRepl)
517 pkg.DownloadURL = strings.ReplaceAll(pkg.DownloadURL, "|", pipeRepl)
518 for i := range pkg.WebURLs {
519 pkg.WebURLs[i] = strings.ReplaceAll(pkg.WebURLs[i], "|", pipeRepl)
520 }
521}
522
523func genAMMeta(filename string, metadata DbinMetadata) {
524 replaceEmptyWithNil := func(value string) string {
525 if value == "" {
526 return "nil"
527 }
528 return value
529 }
530
531 file, err := os.Create(filename + ".txt")
532 if err != nil {
533 fmt.Println("Error creating output file:", err)
534 return
535 }
536 defer file.Close()
537
538 for _, items := range metadata {
539 for _, pkg := range items {
540 pkg.Name = replaceEmptyWithNil(pkg.Name)
541 pkg.Description = replaceEmptyWithNil(pkg.Description)
542 pkg.DownloadURL = replaceEmptyWithNil(pkg.DownloadURL)
543
544 webURL := pkg.DownloadURL
545 if webURL == "nil" && len(pkg.WebURLs) > 0 {
546 webURL = pkg.WebURLs[0]
547 }
548
549 replacePipeFields(&pkg)
550
551 pkgName := pkg.Name
552 strings.ToLower(pkgName)
553 strings.ReplaceAll(pkgName, " ", "-")
554
555 bsum := pkg.Bsum
556 if len(bsum) > 12 {
557 bsum = bsum[:12]
558 } else {
559 bsum = "nil"
560 }
561
562 file.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s |\n",
563 pkgName, pkg.Description, webURL, pkg.DownloadURL, bsum))
564 }
565 }
566}
567*/