Author: Adam <git@apiote.tk>
add filterable item_cache
db/db.go | 189 +++++++++++++++++++++++++++++++++++++++++++--- libamuse/account.go | 70 +++++++++++++++++ libamuse/common.go | 2 libamuse/film.go | 11 -- tmdb/common.go | 79 +----------------- tmdb/film.go | 45 ++++++++-- tmdb/serie.go | 12 +-
diff --git a/db/db.go b/db/db.go index 0586233dd5dc4fc5802c44d648ae0242512a141a..5b2b4b7f72aca6466979a5a6ba254d0cd922bfbb 100644 --- a/db/db.go +++ b/db/db.go @@ -1,7 +1,6 @@ package db import ( - "notabug.org/apiote/amuse/tmdb" "notabug.org/apiote/amuse/utils" "crypto/rand" @@ -25,6 +24,25 @@ ItemTypeTvserie = "tvserie" ItemTypeUnkown = "unknown" ) +type ItemInfo struct { + Cover string + Status string + Title string + YearStart int + YearEnd int + BasedOn string + Genres string + Runtime int + Collection int + Part int + Item string +} + +type CacheEntry struct { + Etag string + Data []byte +} + type EmptyError struct { message string } @@ -52,7 +70,6 @@ IsLong bool } func Migrate() error { - // todo migrations db, err := sql.Open("sqlite3", utils.DataHome+"/amuse.db") if err != nil { return err @@ -60,23 +77,27 @@ } defer db.Close() _, err = db.Exec(`create table cache(uri text primary key, etag text, date date, response blob, last_hit date)`) - if err != nil { + if err != nil && err.Error() != "table cache already exists" { return err } _, err = db.Exec(`create table users(username text primary key, password text, sfa text, avatar blob, avatar_small blob, is_admin bool, recovery_codes text, timezone text)`) - if err != nil { + if err != nil && err.Error() != "table users already exists" { return err } _, err = db.Exec(`create table sessions(id text primary key, username text, expiry datetime, is_long boolean, foreign key(username) references users(username))`) - if err != nil { + if err != nil && err.Error() != "table sessions already exists" { return err } _, err = db.Exec(`create table wantlist(username text, item_type text, item_id text, primary key(username, item_type, item_id), foreign key(username) references users(username))`) - if err != nil { + if err != nil && err.Error() != "table wantlist already exists" { return err } _, err = db.Exec(`create table experiences(username text, item_type text, item_id text, time datetime, foreign key(username) references users(username), primary key(username, item_type, item_id, time))`) - if err != nil { + if err != nil && err.Error() != "table experiences already exists" { + return err + } + _, err = db.Exec(`create table item_cache (item_type text, item_id text, cover text, status text, title text, year_start int, year_end int, based_on text, genres text, runtime int, collection int, part int, ref_count int, primary key(item_type, item_id))`) + if err != nil && err.Error() != "table item_cache already exists" { return err } return nil @@ -302,6 +323,7 @@ defer db.Close() for e := 1; e <= episodesNumber; e++ { episodeId := fmt.Sprintf("%s/S00E%02d", itemId, e) + // todo if not watched already _, err = db.Exec(`insert into experiences values(?, ?, ?, ?)`, username, itemType, episodeId, datetime) if err != nil { if err.Error()[:6] != "UNIQUE" { @@ -367,12 +389,151 @@ return isOnlist, nil } -func GetItemTypeFromShow(show tmdb.Show) ItemType { - if _, ok := show.(*tmdb.Film); ok { - return ItemTypeFilm - } else if _, ok := show.(*tmdb.TvSerie); ok { - return ItemTypeTvserie - } else { - return ItemTypeUnkown +func SaveCacheItem(itemType ItemType, itemId string, itemInfo ItemInfo) error { + db, err := sql.Open("sqlite3", utils.DataHome+"/amuse.db") + if err != nil { + fmt.Fprintf(os.Stderr, "DB open err\n") + return err + } + defer db.Close() + + _, err = db.Exec(`insert into item_cache values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + on conflict(item_type, item_id) do update set ref_count = ref_count + 1`, + itemType, itemId, itemInfo.Cover, itemInfo.Status, itemInfo.Title, itemInfo.YearStart, itemInfo.YearEnd, itemInfo.BasedOn, itemInfo.Genres, itemInfo.Runtime, itemInfo.Collection, itemInfo.Part, 1) + if err != nil { + return err + } + return nil +} + +func UpdateCacheItem(itemType ItemType, itemId string, itemInfo ItemInfo) error { + db, err := sql.Open("sqlite3", utils.DataHome+"/amuse.db") + if err != nil { + fmt.Fprintf(os.Stderr, "DB open err\n") + return err + } + defer db.Close() + + db.Exec(`update item_cache set cover = ?, status = ?, title = ?, year_start = ?, year_end = ?, based_on = ?, genres = ?, runtime = ?, collection = ?, part = ? where item_type = ? and item_id = ?`, itemInfo.Cover, itemInfo.Status, itemInfo.Title, itemInfo.YearStart, itemInfo.YearEnd, itemInfo.BasedOn, itemInfo.Genres, itemInfo.Runtime, itemInfo.Collection, itemInfo.Part, itemType, itemId) + + return nil +} + +func RemoveCacheItem(itemType ItemType, itemId string) error { + db, err := sql.Open("sqlite3", utils.DataHome+"/amuse.db") + if err != nil { + fmt.Fprintf(os.Stderr, "DB open err\n") + return err + } + defer db.Close() + + _, err = db.Exec(`update item_cache set ref_count = ref_count - 1`) + + return err +} + +func CleanItemCache() error { + db, err := sql.Open("sqlite3", utils.DataHome+"/amuse.db") + if err != nil { + fmt.Fprintf(os.Stderr, "DB open err\n") + return err + } + defer db.Close() + + _, err = db.Exec(`delete from item_cache where ref_count <= 0`) + + return err +} + +func GetCacheItem(itemType ItemType, itemId string) (*ItemInfo, error) { + db, err := sql.Open("sqlite3", utils.DataHome+"/amuse.db") + if err != nil { + fmt.Fprintf(os.Stderr, "DB open err\n") + return nil, err + } + defer db.Close() + + var ( + itemInfo ItemInfo + itemTypeDb ItemType + itemIdDb string + refCount int + ) + + row := db.QueryRow(`select * from cache where item_type = ? and item_id = ?`, itemType, itemId) + + err = row.Scan(&itemTypeDb, &itemIdDb, &itemInfo.Cover, &itemInfo.Status, &itemInfo.Title, &itemInfo.YearStart, &itemInfo.YearEnd, &itemInfo.BasedOn, &itemInfo.Genres, &itemInfo.Runtime, &itemInfo.Collection, &itemInfo.Part, refCount) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } else { + return nil, err + } + } + return &itemInfo, nil +} + +// ==== + +func GetCacheEntry(uri string) (*CacheEntry, error) { + db, err := sql.Open("sqlite3", utils.DataHome+"/amuse.db") + if err != nil { + fmt.Fprintf(os.Stderr, "DB open err\n") + return nil, err } + defer db.Close() + + row := db.QueryRow(`select etag, response from cache where uri = ?`, uri) + + var cacheEntry CacheEntry + err = row.Scan(&cacheEntry.Etag, &cacheEntry.Data) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } else { + return nil, err + } + } + + return &cacheEntry, err +} + +func CleanCache() error { + db, err := sql.Open("sqlite3", utils.DataHome+"/amuse.db") + if err != nil { + fmt.Fprintf(os.Stderr, "DB open err\n") + return err + } + defer db.Close() + + row := db.QueryRow(`select count(*) from cache`) + + var count int + err = row.Scan(&count) + if err != nil { + return err + } + + for count > 10000 { + _, err = db.Exec(`delete from cache where last_update = (select min(last_update) from cache)`) + if err != nil { + return err + } + count-- + } + + return nil +} + +func SaveCacheEntry(uri, etag string, data []byte) error { + db, err := sql.Open("sqlite3", utils.DataHome+"/amuse.db") + if err != nil { + fmt.Fprintf(os.Stderr, "DB open err\n") + return err + } + defer db.Close() + + _, err = db.Exec(`insert into cache values(?, ?, null, ?, datetime('now')) + on conflict(uri) do update set etag = excluded.etag, response = excluded.response, last_hit = excluded.last_hit`, uri, etag, data) + return err } diff --git a/libamuse/account.go b/libamuse/account.go index 7ac1346ea4b25c6a8ff65f674b6cc77302db1ad7..8f372db794ffc0ae6e0703397ced590f8ffbb6fc 100644 --- a/libamuse/account.go +++ b/libamuse/account.go @@ -3,6 +3,7 @@ import ( "notabug.org/apiote/amuse/accounts" "notabug.org/apiote/amuse/db" + "notabug.org/apiote/amuse/tmdb" "errors" "fmt" @@ -56,6 +57,63 @@ return gott.Tuple(args), err } +func getItem(args ...interface{}) (interface{}, error) { + itemType := args[2].(string) + var arg interface{} + var err error + switch itemType { + case db.ItemTypeFilm: + arg, err = gott. + NewResult(gott.Tuple(args)). + Bind(getFilm). + Bind(getCollection). + Finish() + args = arg.(gott.Tuple) + break + } + return gott.Tuple(args), err +} + +func cacheItemFilm(film *tmdb.Film) db.ItemInfo { // todo -> interface Item + part := 0 + for i, p := range film.Collection.Parts { + if p.Title == film.Title { + part = i + break + } + } + genres := "" + for _, genre := range film.Genres { + genres += fmt.Sprintf("%d", genre.Id) + "," + } + + itemInfo := db.ItemInfo{ + Cover: film.Poster_path, + Status: film.Status, + Title: film.Original_title, + YearStart: film.Release_date.Year(), + // todo BasedOn: + Genres: genres, + Runtime: film.Runtime, + Collection: film.Collection.Id, + Part: part, + } + return itemInfo +} + +func cacheItem(args ...interface{}) (interface{}, error) { + data := args[0].(*RequestData) + result := args[1].(*Result) + var itemInfo db.ItemInfo + + if film, ok := result.result.(*tmdb.Film); ok { + itemInfo = cacheItemFilm(film) + } + + err := db.SaveCacheItem(db.ItemTypeFilm, data.id, itemInfo) + return gott.Tuple(args), err +} + func AddToWantlist(username string, auth accounts.Authentication, itemId, itemType, language, mimetype string) error { auth.Necessary = true _, err := gott. @@ -63,6 +121,8 @@ NewResult(gott.Tuple{&RequestData{id: itemId, language: language, mimetype: mimetype, auth: auth, username: username}, &Result{}, itemType}). Bind(parseLanguage). Bind(verifyToken). Bind(verifyUser). + Bind(getItem). + Bind(cacheItem). Bind(addToWantlist). Finish() @@ -117,6 +177,15 @@ return gott.Tuple(args), err } +func removeCacheItem(args ...interface{}) (interface{}, error) { + data := args[0].(*RequestData) + itemType := args[2].(string) + + err := db.RemoveCacheItem(db.ItemType(itemType), data.id) + + return gott.Tuple(args), err +} + func AddToExperiences(username string, auth accounts.Authentication, itemId, itemType, datetime, language, mimetype string) error { auth.Necessary = true _, err := gott. @@ -127,6 +196,7 @@ Bind(verifyUser). Bind(parseExperienceDate). Bind(addToExperiences). Bind(removeFromWantList). + Bind(removeCacheItem). Finish() return err diff --git a/libamuse/common.go b/libamuse/common.go index 4d53a3cba6018562e5f4b0da6632dc3c5457a77d..ddd1dde94735b8d8668e09291b90603e38d5a8f7 100644 --- a/libamuse/common.go +++ b/libamuse/common.go @@ -102,7 +102,7 @@ if result.user.IsEmpty() { return gott.Tuple(args), nil } - itemType := db.GetItemTypeFromShow(show) + itemType := tmdb.GetItemTypeFromShow(show) isOnList, err := db.IsOnWantList(result.user.Username, data.id, itemType) show.SetOnWantList(isOnList) diff --git a/libamuse/film.go b/libamuse/film.go index e1c3d3e443b0c1db2e3fa221e9ebbbc1650edaf6..1324e42cf47d07ca302f7c966155ff696142e63e 100644 --- a/libamuse/film.go +++ b/libamuse/film.go @@ -14,13 +14,12 @@ func getFilm(args ...interface{}) (interface{}, error) { data := args[0].(*RequestData) result := args[1].(*Result) languages := result.languages - film, err := tmdb.GetFilm(data.id, languages[0].String(), data.connection) + film, err := tmdb.GetFilm(data.id, languages[0].String()) result.result = film return gott.Tuple(args), err } func getCollection(args ...interface{}) (interface{}, error) { - data := args[0].(*RequestData) result := args[1].(*Result) film := result.result.(*tmdb.Film) languages := result.languages @@ -28,8 +27,7 @@ var err error if film.Collection.Id != 0 { collection, e := tmdb.GetCollection( strconv.FormatInt(int64(film.Collection.Id), 10), - languages[0].String(), - data.connection) + languages[0].String()) film.Collection = *collection err = e } @@ -62,7 +60,6 @@ auth.Necessary = false request := &RequestData{id: id, language: language, mimetype: mimetype, auth: auth} r, err := gott. NewResult(gott.Tuple{request, &Result{}}). - Bind(createDbConnection). Bind(parseLanguage). Bind(verifyToken). Bind(getFilm). @@ -73,10 +70,6 @@ Bind(isOnWantList). Bind(createRenderer). Map(renderFilm). Finish() - - if request.connection != nil { - request.connection.Close() - } if err != nil { return "", err diff --git a/tmdb/common.go b/tmdb/common.go index d4f347f89631a842adb1e513cd6edc8122fcc8fa..27e9500812174ce07b44181da67402299a0d1919 100644 --- a/tmdb/common.go +++ b/tmdb/common.go @@ -1,10 +1,8 @@ package tmdb import ( - "notabug.org/apiote/amuse/network" "notabug.org/apiote/amuse/wikidata" - - "notabug.org/apiote/gott" + "notabug.org/apiote/amuse/db" ) const ( @@ -35,74 +33,13 @@ Cast []ShowCastEntry Crew []ShowCrewEntry } -func getCacheEntry(args ...interface{}) (interface{}, error) { - request := args[0].(*network.Request) - result := args[1].(*network.Result) - uri := result.Request.URL.String() - - rows, err := request.Connection.Query(`select etag, response from cache where uri = ?`, uri) - if err != nil { - return gott.Tuple(args), err - } - defer rows.Close() - - var ( - etag string - data []byte - ) - if rows.Next() { - if err = rows.Scan(&etag, &data); err != nil { - return gott.Tuple(args), err - } +func GetItemTypeFromShow(show Show) db.ItemType { + if _, ok := show.(*Film); ok { + return db.ItemTypeFilm + } else if _, ok := show.(*TvSerie); ok { + return db.ItemTypeTvserie + } else { + return db.ItemTypeUnkown } - request.Etag = etag - result.Body = data - return gott.Tuple(args), nil } -func cleanCache(args ...interface{}) (interface{}, error) { - request := args[0].(*network.Request) - rows, err := request.Connection.Query(`select count(*) from cache`) - if err != nil { - return gott.Tuple(args), err - } - defer rows.Close() - rows.Next() - var n int - err = rows.Scan(&n) - if err != nil { - return gott.Tuple(args), err - } - for n > 10000 { - _, err = request.Connection.Exec(`delete from cache where last_hit = (select min(last_hit) from cache)`) - if err != nil { - return gott.Tuple(args), err - } - n-- - } - return gott.Tuple(args), nil -} - -func saveCacheEntry(args ...interface{}) (interface{}, error) { - request := args[0].(*network.Request) - result := args[1].(*network.Result) - uri := result.Request.URL.String() - if result.Etag != "" { - body := []byte(result.Body) - etag := result.Etag - - _, err := request.Connection.Exec(`insert into cache values(?, ?, null, ?, datetime('now')) - on conflict(uri) do update set etag = ?, response = ?`, uri, etag, body, etag, body) - if err != nil { - return gott.Tuple(args), err - } - } else { - etag := request.Etag - _, err := request.Connection.Exec(`update cache set last_hit = datetime('now') - where uri = ? and etag = ?`, uri, etag) - if err != nil { - return gott.Tuple(args), err - } - } - return gott.Tuple(args), nil -} diff --git a/tmdb/film.go b/tmdb/film.go index 0b3aae17a761083a6fd62126a52573a51f520ead..d6da001b0ad1b6c803a8dabbb10040d16facf64d 100644 --- a/tmdb/film.go +++ b/tmdb/film.go @@ -1,12 +1,12 @@ package tmdb import ( + "notabug.org/apiote/amuse/db" "notabug.org/apiote/amuse/i18n" "notabug.org/apiote/amuse/network" "notabug.org/apiote/amuse/utils" "notabug.org/apiote/amuse/wikidata" - "database/sql" "encoding/json" "net/http" "sort" @@ -33,6 +33,7 @@ Etag string Backdrop_path string `json:"backdrop_path"` Collection Collection `json:"belongs_to_collection"` Genres []struct { + Id int Name string } Original_title string `json:"original_title"` @@ -84,6 +85,7 @@ result := args[1].(*network.Result) film := &Film{} err := json.Unmarshal(result.Body, film) film.Source = "https://www.themoviedb.org/movie/" + id + film.Etag = result.Etag result.Result = film return gott.Tuple(args), err } @@ -141,17 +143,42 @@ }) return gott.Tuple(args) } -func GetFilm(id, language string, connection *sql.DB) (*Film, error) { +func getCacheEntry(args ...interface{}) (interface{}, error) { + result := args[1].(*network.Result) + uri := result.Request.URL.String() + entry, err := db.GetCacheEntry(uri) + if err != nil || entry == nil { + return gott.Tuple(args), err + } + + result.Etag = entry.Etag + result.Body = entry.Data + return gott.Tuple(args), nil +} + +func cleanCache(args ...interface{}) error { + err := db.CleanCache() + return err +} + +func saveCacheEntry(args ...interface{}) error { + result := args[1].(*network.Result) + uri := result.Request.URL.String() + err := db.SaveCacheEntry(uri, result.Etag, result.Body) + return err +} + +func GetFilm(id, language string) (*Film, error) { film, err := gott. - NewResult(gott.Tuple{&network.Request{Id: id, Language: language, Connection: connection}, &network.Result{}}). + NewResult(gott.Tuple{&network.Request{Id: id, Language: language}, &network.Result{}}). Bind(createFilmRequest). Bind(getCacheEntry). Map(network.AddHeaders). Bind(network.DoRequest). Bind(network.HandleRequestError). Bind(network.ReadResponse). - Bind(cleanCache). - Bind(saveCacheEntry). + Tee(cleanCache). + Tee(saveCacheEntry). Bind(unmarshalFilm). Bind(convertFilmDate). Finish() @@ -163,17 +190,17 @@ return film.(gott.Tuple)[1].(*network.Result).Result.(*Film), nil } } -func GetCollection(id, language string, connection *sql.DB) (*Collection, error) { +func GetCollection(id, language string) (*Collection, error) { collection, err := gott. - NewResult(gott.Tuple{&network.Request{Id: id, Language: language, Connection: connection}, &network.Result{}}). + NewResult(gott.Tuple{&network.Request{Id: id, Language: language}, &network.Result{}}). Bind(createCollectionRequest). Bind(getCacheEntry). Map(network.AddHeaders). Bind(network.DoRequest). Bind(network.HandleRequestError). Bind(network.ReadResponse). - Bind(cleanCache). - Bind(saveCacheEntry). + Tee(cleanCache). + Tee(saveCacheEntry). Bind(unmarshalCollection). Bind(convertCollectionDates). Map(sortCollection). diff --git a/tmdb/serie.go b/tmdb/serie.go index d37af3580968a731af5f4062243f1228db0b38b8..f3f460f804b1acee62b1b2a6a61aec8fc14005d4 100644 --- a/tmdb/serie.go +++ b/tmdb/serie.go @@ -181,13 +181,13 @@ func GetSerie(id, language string, connection *sql.DB) (*TvSerie, error) { serie, err := gott. NewResult(gott.Tuple{&network.Request{Id: id, Language: language, Connection: connection}, &network.Result{}}). Bind(createSerieRequest). - Bind(getCacheEntry). + //Bind(getCacheEntry). Map(network.AddHeaders). Bind(network.DoRequest). Bind(network.HandleRequestError). Bind(network.ReadResponse). - Bind(cleanCache). - Bind(saveCacheEntry). + //Bind(cleanCache). + //Bind(saveCacheEntry). Bind(unmarshalSerie). Bind(convertSerieDates). Finish() @@ -207,13 +207,13 @@ seasonNumber := strconv.FormatInt(int64(season.Season_number), 10) s, err2 := gott. NewResult(gott.Tuple{&network.Request{Id: serie.Id, Language: language, Subid: seasonNumber, Connection: connection}, &network.Result{}}). Bind(createSeasonRequest). - Bind(getCacheEntry). + //Bind(getCacheEntry). Map(network.AddHeaders). Bind(network.DoRequest). Bind(network.HandleRequestError). Bind(network.ReadResponse). - Bind(cleanCache). - Bind(saveCacheEntry). + //Bind(cleanCache). + //Bind(saveCacheEntry). Bind(unmarshalSeason). Bind(convertSeasonDates). Finish()