import stashapi.log as log from stashapi.stashapp import StashInterface from stashapi.stashbox import StashBoxInterface import os import sys import requests import json import requests from pathlib import Path import base64 per_page = 100 request_s = requests.Session() stash_boxes = {} scrapers = {} def processImages(img): log.debug("image: %s" % (img,)) image_data = None for file in [x["path"] for x in img["visual_files"]]: if settings["path"] in file: index_file = Path(Path(file).parent) / (Path(file).stem + ".json") log.debug(index_file) if index_file.exists(): log.debug("loading index file %s" % (index_file,)) with open(index_file) as f: index = json.load(f) index["id"] = img["id"] if image_data: image_data["gallery_ids"].extend(index["gallery_ids"]) else: image_data = index if image_data: # log.debug(image_data) stash.update_image(image_data) def processPerformers(): query = { "tags": { "depth": 0, "excludes": [], "modifier": "INCLUDES_ALL", "value": [tag_stashbox_performer_gallery], } } performers = stash.find_performers(f=query) for performer in performers: processPerformer(performer) def processPerformer(performer): dir = Path(settings["path"]) / performer["id"] dir.mkdir(parents=True, exist_ok=True) nogallery = dir / ".nogallery" nogallery.touch() for sid in performer["stash_ids"]: log.debug(sid) processPerformerStashid(sid["endpoint"], sid["stash_id"], performer) def get_stashbox(endpoint): for sbx_config in stash.get_configuration()["general"]["stashBoxes"]: if sbx_config["endpoint"] == endpoint: stashbox = StashBoxInterface( {"endpoint": sbx_config["endpoint"], "api_key": sbx_config["api_key"]} ) stash_boxes[endpoint] = stashbox return stashbox def processPerformerStashid(endpoint, stashid, p): log.info( "processing performer %s, %s endpoint: %s, stash id: %s" % ( p["name"], p["id"], endpoint, stashid, ) ) index_file = os.path.join(settings["path"], p["id"], "index.json") if os.path.exists(index_file): with open(os.path.join(settings["path"], p["id"], "index.json")) as f: index = json.load(f) else: index = {"files": {}, "galleries": {}, "performer_id": p["id"]} modified = False stashbox = get_stashbox(endpoint) if stashbox: query = """id name images { id url } urls{ url type } """ perf = stashbox.find_performer(stashid, fragment=query) log.debug(perf) if endpoint not in index["galleries"]: gallery_input = { "title": "%s - %s " % ( p["name"], endpoint[8:-8], ), "urls": [ "%s/performers/%s" % ( endpoint[:-8], stashid, ) ], "tag_ids": [tag_stashbox_performer_gallery], "performer_ids": [p["id"]], } gal = stash.create_gallery(gallery_input) log.info("Created gallery %s" % (gal,)) index["galleries"][endpoint] = gal modified = True # check if the gallery still exists and has not been deleted current_gal = stash.find_gallery(index["galleries"][endpoint]) log.debug("current: %s" % (current_gal,)) if current_gal is None: log.debug("deleted?") gallery_input = { "title": "%s - %s " % ( p["name"], endpoint[:-8], ), "urls": [ "%s/performers/%s" % ( endpoint[:-8], stashid, ) ], "tag_ids": [tag_stashbox_performer_gallery], "performer_ids": [p["id"]], } gal = stash.create_gallery(gallery_input) log.info("Created gallery %s" % (gal,)) index["galleries"][endpoint] = gal modified = True if modified: with open(index_file, "w") as f: json.dump(index, f) for img in perf["images"]: image_index = Path(settings["path"]) / p["id"] / (img["id"] + ".json") if not image_index.exists(): with open(image_index, "w") as f: image_data = { "title": img["id"], "urls": [img["url"]], "performer_ids": [p["id"]], "tag_ids": [tag_stashbox_performer_gallery], "gallery_ids": [index["galleries"][endpoint]], } json.dump(image_data, f) filename = Path(settings["path"]) / p["id"] / (img["id"] + ".jpg") if not os.path.exists(filename): log.info( "Downloading image %s to %s" % ( img["url"], filename, ) ) r = requests.get(img["url"]) with open(filename, "wb") as f: f.write(r.content) f.close() # modified=True else: log.debug("image already downloaded") # scrape urls on the performer using the url scrapers in stash if settings["runPerformerScraper"] and len(perf["urls"]) > 0: # we need to determine what scrapers we have and what url patterns they accept, query what url patterns are supported, should only need to check once if len(scrapers) == 0: scrapers_graphql = """query ListPerformerScrapers { listScrapers(types: [PERFORMER]) { id name performer { urls supported_scrapes } } }""" res = stash.callGQL(scrapers_graphql) for r in res["listScrapers"]: if r["performer"]["urls"]: for url in r["performer"]["urls"]: scrapers[url] = r for u in perf["urls"]: for url in scrapers.keys(): if url in u["url"]: log.info( "Running stash scraper on performer url: %s" % (u["url"],) ) res = stash.scrape_performer_url(u["url"]) # Check if the scraper returned a result if res is not None: log.debug(res) # it's possible for multiple images to be returned by a scraper so incriment a number each image image_id = 1 if res["images"]: for image in res["images"]: image_index = ( Path(settings["path"]) / p["id"] / ( "%s-%s.json" % ( scrapers[url]["id"], image_id, ) ) ) if not image_index.exists(): with open(image_index, "w") as f: image_data = { "title": "%s - %s " % ( scrapers[url]["id"], image_id, ), "details": "name: %s\ngender: %s\nurl: %s\ntwitter: %s\ninstagram: %s\nbirthdate: %s\nethnicity: %s\ncountry: %s\neye_color: %s\nheight: %s\nmeasurements: %s\nfake tits: %s\npenis_length: %s\n career length: %s\ntattoos: %s\npiercings: %s\nhair_color: %s\nweight: %s\n description: %s\n" % ( res["name"], res["gender"], res["url"], res["twitter"], res["instagram"], res["birthdate"], res["ethnicity"], res["country"], res["eye_color"], res["height"], res["measurements"], res["fake_tits"], res["penis_length"], res["career_length"], res["tattoos"], res["piercings"], res["hair_color"], res["weight"], res["details"], ), "urls": [ u["url"], ], "performer_ids": [p["id"]], "tag_ids": [ tag_stashbox_performer_gallery ], "gallery_ids": [ index["galleries"][endpoint] ], } json.dump(image_data, f) filename = ( Path(settings["path"]) / p["id"] / ( "%s-%s.jpg" % ( scrapers[url]["id"], image_id, ) ) ) if not filename.exists(): if image.startswith("data:"): with open(filename, "wb") as f: f.write( base64.b64decode( image.split("base64,")[1] ) ) f.close() else: with open(image_index, "w") as f: image_data = { "title": "%s - %s " % ( scrapers[url]["id"], image_id, ), "details": "%s" % (res,), "urls": [u["url"], image], "performer_ids": [p["id"]], "tag_ids": [ tag_stashbox_performer_gallery ], "gallery_ids": [ index["galleries"][endpoint] ], } json.dump(image_data, f) filename = ( Path(settings["path"]) / p["id"] / ("%s.jpg" % (image_id,)) ) r = requests.get(img["url"]) if r.status_code == 200: with open(filename, "wb") as f: f.write(r.content) f.close() image_id = image_id + 1 # log.debug('%s %s' % (url['url'],url['type'],)) # stash.scraper # scrape=stash.scrape_performer_url(ur) else: log.error("endpoint %s not configured, skipping" % (endpoint,)) def setPerformerPicture(img): if len(img["performers"]) == 1: log.debug(img["paths"]["image"]) res = request_s.get(img["paths"]["image"]) log.debug(res.headers["Content-Type"]) if res.status_code == 200: encoded = base64.b64encode(res.content).decode() new_performer = { "id": img["performers"][0]["id"], "image": "data:{0};base64,{1}".format( res.headers["Content-Type"], encoded ), } log.info("updating performer with tagged image %s" % (new_performer["id"],)) stash.update_performer(new_performer) def processQueue(): for id in settings["queue"].split(","): if len(id) > 0: p = stash.find_performer(id) processPerformer(p) # queue has not changed since we started, clear setting if ( stash.get_configuration()["plugins"]["stashdb-performer-gallery"] == settings["queue"] ): stash.configure_plugin("stashdb-performer-gallery", {"queue": ""}) stash.metadata_scan(paths=[settings["path"]]) stash.run_plugin_task("stashdb-performer-gallery", "relink missing images") else: # update remove the completed entries from the queue string leaving the unprocessed and schedule the task again log.debug("updating queue") stash.configure_plugin( "stashdb-performer-gallery", { "queue": stash.get_configuration()["plugins"][ "stashdb-performer-gallery" ]["queue"].removeprefix(settings["queue"]) }, ) stash.run_plugin_task( "stashdb-performer-gallery", "Process Performers", args={"full": False} ) def relink_images(performer_id=None): query = { "path": {"modifier": "INCLUDES", "value": settings["path"]}, } if performer_id == None: query["is_missing"] = "galleries" query["path"] = {"modifier": "INCLUDES", "value": settings["path"]} else: query["path"] = { "modifier": "INCLUDES", "value": str(Path(settings["path"]) / performer_id / ""), } # else: # query["file_count"] = {"modifier": "NOT_EQUALS", "value": 1} total = stash.find_images(f=query, get_count=True)[0] i = 0 images = [] while i < total: images = stash.find_images(f=query, filter={"page": i, "per_page": per_page}) for img in images: log.debug("image: %s" % (img,)) processImages(img) i = i + 1 log.progress((i / total)) json_input = json.loads(sys.stdin.read()) FRAGMENT_SERVER = json_input["server_connection"] stash = StashInterface(FRAGMENT_SERVER) config = stash.get_configuration()["plugins"] settings = { "path": "/download_dir", "runPerformerScraper": False, } if "stashdb-performer-gallery" in config: settings.update(config["stashdb-performer-gallery"]) # log.info('config: %s ' % (settings,)) tag_stashbox_performer_gallery = stash.find_tag( "[Stashbox Performer Gallery]", create=True ).get("id") tag_performer_image = stash.find_tag("[Set Profile Image]", create=True).get("id") if "stasdb-performer-gallery" in config: settings.update(config["stasdb-performer-gallery"]) if "mode" in json_input["args"]: PLUGIN_ARGS = json_input["args"]["mode"] if "performer" in json_input["args"]: p = stash.find_performer(json_input["args"]["performer"]) if tag_stashbox_performer_gallery in [x["id"] for x in p["tags"]]: processPerformer(p) stash.metadata_scan(paths=[settings["path"]]) stash.run_plugin_task( "stashdb-performer-gallery", "relink missing images", args={"performer_id": p["id"]}, ) elif "processPerformers" in PLUGIN_ARGS: processPerformers() stash.metadata_scan([settings["path"]]) stash.run_plugin_task( "stashdb-performer-gallery", "relink missing images", args={} ) elif "processImages" in PLUGIN_ARGS: if "performer_id" in json_input["args"]: relink_images(performer_id=json_input["args"]["performer_id"]) else: relink_images() elif "hookContext" in json_input["args"]: id = json_input["args"]["hookContext"]["id"] if json_input["args"]["hookContext"]["type"] == "Image.Create.Post": img = stash.find_image(image_in=id) processImages(img) if json_input["args"]["hookContext"]["type"] == "Image.Update.Post": img = stash.find_image(image_in=id) if tag_performer_image in [x["id"] for x in img["tags"]]: setPerformerPicture(img) if json_input["args"]["hookContext"]["type"] == "Performer.Update.Post": stash.run_plugin_task( "stashdb-performer-gallery", "Process Performers", args={"performer": id} )