stash
This commit is contained in:
Christoph Califice
@@ -0,0 +1,470 @@
|
||||
import stashapi.log as log
|
||||
from stashapi.stashapp import StashInterface
|
||||
from stashapi.stashbox import StashBoxInterface
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
import json
|
||||
import requests
|
||||
from pathlib import Path
|
||||
import base64
|
||||
|
||||
|
||||
per_page = 100
|
||||
request_s = requests.Session()
|
||||
stash_boxes = {}
|
||||
scrapers = {}
|
||||
|
||||
|
||||
def processImages(img):
|
||||
log.debug("image: %s" % (img,))
|
||||
image_data = None
|
||||
for file in [x["path"] for x in img["visual_files"]]:
|
||||
if settings["path"] in file:
|
||||
index_file = Path(Path(file).parent) / (Path(file).stem + ".json")
|
||||
log.debug(index_file)
|
||||
if index_file.exists():
|
||||
log.debug("loading index file %s" % (index_file,))
|
||||
with open(index_file) as f:
|
||||
index = json.load(f)
|
||||
index["id"] = img["id"]
|
||||
if image_data:
|
||||
image_data["gallery_ids"].extend(index["gallery_ids"])
|
||||
else:
|
||||
image_data = index
|
||||
if image_data:
|
||||
# log.debug(image_data)
|
||||
stash.update_image(image_data)
|
||||
|
||||
|
||||
def processPerformers():
|
||||
query = {
|
||||
"tags": {
|
||||
"depth": 0,
|
||||
"excludes": [],
|
||||
"modifier": "INCLUDES_ALL",
|
||||
"value": [tag_stashbox_performer_gallery],
|
||||
}
|
||||
}
|
||||
performers = stash.find_performers(f=query)
|
||||
|
||||
for performer in performers:
|
||||
processPerformer(performer)
|
||||
|
||||
|
||||
def processPerformer(performer):
|
||||
dir = Path(settings["path"]) / performer["id"]
|
||||
dir.mkdir(parents=True, exist_ok=True)
|
||||
nogallery = dir / ".nogallery"
|
||||
nogallery.touch()
|
||||
for sid in performer["stash_ids"]:
|
||||
log.debug(sid)
|
||||
processPerformerStashid(sid["endpoint"], sid["stash_id"], performer)
|
||||
|
||||
|
||||
def get_stashbox(endpoint):
|
||||
for sbx_config in stash.get_configuration()["general"]["stashBoxes"]:
|
||||
if sbx_config["endpoint"] == endpoint:
|
||||
stashbox = StashBoxInterface(
|
||||
{"endpoint": sbx_config["endpoint"], "api_key": sbx_config["api_key"]}
|
||||
)
|
||||
stash_boxes[endpoint] = stashbox
|
||||
return stashbox
|
||||
|
||||
|
||||
def processPerformerStashid(endpoint, stashid, p):
|
||||
log.info(
|
||||
"processing performer %s, %s endpoint: %s, stash id: %s"
|
||||
% (
|
||||
p["name"],
|
||||
p["id"],
|
||||
endpoint,
|
||||
stashid,
|
||||
)
|
||||
)
|
||||
|
||||
index_file = os.path.join(settings["path"], p["id"], "index.json")
|
||||
if os.path.exists(index_file):
|
||||
with open(os.path.join(settings["path"], p["id"], "index.json")) as f:
|
||||
index = json.load(f)
|
||||
else:
|
||||
index = {"files": {}, "galleries": {}, "performer_id": p["id"]}
|
||||
|
||||
modified = False
|
||||
stashbox = get_stashbox(endpoint)
|
||||
if stashbox:
|
||||
query = """id
|
||||
name
|
||||
images {
|
||||
id
|
||||
url
|
||||
}
|
||||
urls{
|
||||
url
|
||||
type
|
||||
}
|
||||
"""
|
||||
perf = stashbox.find_performer(stashid, fragment=query)
|
||||
log.debug(perf)
|
||||
if endpoint not in index["galleries"]:
|
||||
gallery_input = {
|
||||
"title": "%s - %s "
|
||||
% (
|
||||
p["name"],
|
||||
endpoint[8:-8],
|
||||
),
|
||||
"urls": [
|
||||
"%s/performers/%s"
|
||||
% (
|
||||
endpoint[:-8],
|
||||
stashid,
|
||||
)
|
||||
],
|
||||
"tag_ids": [tag_stashbox_performer_gallery],
|
||||
"performer_ids": [p["id"]],
|
||||
}
|
||||
gal = stash.create_gallery(gallery_input)
|
||||
log.info("Created gallery %s" % (gal,))
|
||||
index["galleries"][endpoint] = gal
|
||||
|
||||
modified = True
|
||||
# check if the gallery still exists and has not been deleted
|
||||
current_gal = stash.find_gallery(index["galleries"][endpoint])
|
||||
log.debug("current: %s" % (current_gal,))
|
||||
if current_gal is None:
|
||||
log.debug("deleted?")
|
||||
gallery_input = {
|
||||
"title": "%s - %s "
|
||||
% (
|
||||
p["name"],
|
||||
endpoint[:-8],
|
||||
),
|
||||
"urls": [
|
||||
"%s/performers/%s"
|
||||
% (
|
||||
endpoint[:-8],
|
||||
stashid,
|
||||
)
|
||||
],
|
||||
"tag_ids": [tag_stashbox_performer_gallery],
|
||||
"performer_ids": [p["id"]],
|
||||
}
|
||||
|
||||
gal = stash.create_gallery(gallery_input)
|
||||
log.info("Created gallery %s" % (gal,))
|
||||
index["galleries"][endpoint] = gal
|
||||
modified = True
|
||||
if modified:
|
||||
with open(index_file, "w") as f:
|
||||
json.dump(index, f)
|
||||
|
||||
for img in perf["images"]:
|
||||
image_index = Path(settings["path"]) / p["id"] / (img["id"] + ".json")
|
||||
if not image_index.exists():
|
||||
with open(image_index, "w") as f:
|
||||
image_data = {
|
||||
"title": img["id"],
|
||||
"urls": [img["url"]],
|
||||
"performer_ids": [p["id"]],
|
||||
"tag_ids": [tag_stashbox_performer_gallery],
|
||||
"gallery_ids": [index["galleries"][endpoint]],
|
||||
}
|
||||
json.dump(image_data, f)
|
||||
filename = Path(settings["path"]) / p["id"] / (img["id"] + ".jpg")
|
||||
if not os.path.exists(filename):
|
||||
log.info(
|
||||
"Downloading image %s to %s"
|
||||
% (
|
||||
img["url"],
|
||||
filename,
|
||||
)
|
||||
)
|
||||
r = requests.get(img["url"])
|
||||
with open(filename, "wb") as f:
|
||||
f.write(r.content)
|
||||
f.close()
|
||||
# modified=True
|
||||
else:
|
||||
log.debug("image already downloaded")
|
||||
|
||||
# scrape urls on the performer using the url scrapers in stash
|
||||
if settings["runPerformerScraper"] and len(perf["urls"]) > 0:
|
||||
|
||||
# we need to determine what scrapers we have and what url patterns they accept, query what url patterns are supported, should only need to check once
|
||||
if len(scrapers) == 0:
|
||||
scrapers_graphql = """query ListPerformerScrapers {
|
||||
listScrapers(types: [PERFORMER]) {
|
||||
id
|
||||
name
|
||||
performer {
|
||||
urls
|
||||
supported_scrapes
|
||||
}
|
||||
}
|
||||
}"""
|
||||
res = stash.callGQL(scrapers_graphql)
|
||||
for r in res["listScrapers"]:
|
||||
if r["performer"]["urls"]:
|
||||
for url in r["performer"]["urls"]:
|
||||
scrapers[url] = r
|
||||
|
||||
for u in perf["urls"]:
|
||||
for url in scrapers.keys():
|
||||
if url in u["url"]:
|
||||
log.info(
|
||||
"Running stash scraper on performer url: %s" % (u["url"],)
|
||||
)
|
||||
res = stash.scrape_performer_url(u["url"])
|
||||
# Check if the scraper returned a result
|
||||
if res is not None:
|
||||
log.debug(res)
|
||||
# it's possible for multiple images to be returned by a scraper so incriment a number each image
|
||||
image_id = 1
|
||||
if res["images"]:
|
||||
for image in res["images"]:
|
||||
image_index = (
|
||||
Path(settings["path"])
|
||||
/ p["id"]
|
||||
/ (
|
||||
"%s-%s.json"
|
||||
% (
|
||||
scrapers[url]["id"],
|
||||
image_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
if not image_index.exists():
|
||||
with open(image_index, "w") as f:
|
||||
image_data = {
|
||||
"title": "%s - %s "
|
||||
% (
|
||||
scrapers[url]["id"],
|
||||
image_id,
|
||||
),
|
||||
"details": "name: %s\ngender: %s\nurl: %s\ntwitter: %s\ninstagram: %s\nbirthdate: %s\nethnicity: %s\ncountry: %s\neye_color: %s\nheight: %s\nmeasurements: %s\nfake tits: %s\npenis_length: %s\n career length: %s\ntattoos: %s\npiercings: %s\nhair_color: %s\nweight: %s\n description: %s\n"
|
||||
% (
|
||||
res["name"],
|
||||
res["gender"],
|
||||
res["url"],
|
||||
res["twitter"],
|
||||
res["instagram"],
|
||||
res["birthdate"],
|
||||
res["ethnicity"],
|
||||
res["country"],
|
||||
res["eye_color"],
|
||||
res["height"],
|
||||
res["measurements"],
|
||||
res["fake_tits"],
|
||||
res["penis_length"],
|
||||
res["career_length"],
|
||||
res["tattoos"],
|
||||
res["piercings"],
|
||||
res["hair_color"],
|
||||
res["weight"],
|
||||
res["details"],
|
||||
),
|
||||
"urls": [
|
||||
u["url"],
|
||||
],
|
||||
"performer_ids": [p["id"]],
|
||||
"tag_ids": [
|
||||
tag_stashbox_performer_gallery
|
||||
],
|
||||
"gallery_ids": [
|
||||
index["galleries"][endpoint]
|
||||
],
|
||||
}
|
||||
json.dump(image_data, f)
|
||||
filename = (
|
||||
Path(settings["path"])
|
||||
/ p["id"]
|
||||
/ (
|
||||
"%s-%s.jpg"
|
||||
% (
|
||||
scrapers[url]["id"],
|
||||
image_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
if not filename.exists():
|
||||
if image.startswith("data:"):
|
||||
with open(filename, "wb") as f:
|
||||
f.write(
|
||||
base64.b64decode(
|
||||
image.split("base64,")[1]
|
||||
)
|
||||
)
|
||||
f.close()
|
||||
else:
|
||||
with open(image_index, "w") as f:
|
||||
image_data = {
|
||||
"title": "%s - %s "
|
||||
% (
|
||||
scrapers[url]["id"],
|
||||
image_id,
|
||||
),
|
||||
"details": "%s" % (res,),
|
||||
"urls": [u["url"], image],
|
||||
"performer_ids": [p["id"]],
|
||||
"tag_ids": [
|
||||
tag_stashbox_performer_gallery
|
||||
],
|
||||
"gallery_ids": [
|
||||
index["galleries"][endpoint]
|
||||
],
|
||||
}
|
||||
json.dump(image_data, f)
|
||||
filename = (
|
||||
Path(settings["path"])
|
||||
/ p["id"]
|
||||
/ ("%s.jpg" % (image_id,))
|
||||
)
|
||||
r = requests.get(img["url"])
|
||||
if r.status_code == 200:
|
||||
with open(filename, "wb") as f:
|
||||
f.write(r.content)
|
||||
f.close()
|
||||
image_id = image_id + 1
|
||||
|
||||
# log.debug('%s %s' % (url['url'],url['type'],))
|
||||
# stash.scraper
|
||||
# scrape=stash.scrape_performer_url(ur)
|
||||
|
||||
else:
|
||||
log.error("endpoint %s not configured, skipping" % (endpoint,))
|
||||
|
||||
|
||||
def setPerformerPicture(img):
|
||||
if len(img["performers"]) == 1:
|
||||
log.debug(img["paths"]["image"])
|
||||
res = request_s.get(img["paths"]["image"])
|
||||
log.debug(res.headers["Content-Type"])
|
||||
if res.status_code == 200:
|
||||
encoded = base64.b64encode(res.content).decode()
|
||||
new_performer = {
|
||||
"id": img["performers"][0]["id"],
|
||||
"image": "data:{0};base64,{1}".format(
|
||||
res.headers["Content-Type"], encoded
|
||||
),
|
||||
}
|
||||
log.info("updating performer with tagged image %s" % (new_performer["id"],))
|
||||
stash.update_performer(new_performer)
|
||||
|
||||
|
||||
def processQueue():
|
||||
for id in settings["queue"].split(","):
|
||||
if len(id) > 0:
|
||||
p = stash.find_performer(id)
|
||||
processPerformer(p)
|
||||
# queue has not changed since we started, clear setting
|
||||
if (
|
||||
stash.get_configuration()["plugins"]["stashdb-performer-gallery"]
|
||||
== settings["queue"]
|
||||
):
|
||||
stash.configure_plugin("stashdb-performer-gallery", {"queue": ""})
|
||||
stash.metadata_scan(paths=[settings["path"]])
|
||||
stash.run_plugin_task("stashdb-performer-gallery", "relink missing images")
|
||||
else:
|
||||
# update remove the completed entries from the queue string leaving the unprocessed and schedule the task again
|
||||
log.debug("updating queue")
|
||||
stash.configure_plugin(
|
||||
"stashdb-performer-gallery",
|
||||
{
|
||||
"queue": stash.get_configuration()["plugins"][
|
||||
"stashdb-performer-gallery"
|
||||
]["queue"].removeprefix(settings["queue"])
|
||||
},
|
||||
)
|
||||
stash.run_plugin_task(
|
||||
"stashdb-performer-gallery", "Process Performers", args={"full": False}
|
||||
)
|
||||
|
||||
|
||||
def relink_images(performer_id=None):
|
||||
query = {
|
||||
"path": {"modifier": "INCLUDES", "value": settings["path"]},
|
||||
}
|
||||
if performer_id == None:
|
||||
query["is_missing"] = "galleries"
|
||||
query["path"] = {"modifier": "INCLUDES", "value": settings["path"]}
|
||||
else:
|
||||
query["path"] = {
|
||||
"modifier": "INCLUDES",
|
||||
"value": str(Path(settings["path"]) / performer_id / ""),
|
||||
}
|
||||
# else:
|
||||
# query["file_count"] = {"modifier": "NOT_EQUALS", "value": 1}
|
||||
|
||||
total = stash.find_images(f=query, get_count=True)[0]
|
||||
i = 0
|
||||
images = []
|
||||
while i < total:
|
||||
images = stash.find_images(f=query, filter={"page": i, "per_page": per_page})
|
||||
for img in images:
|
||||
log.debug("image: %s" % (img,))
|
||||
processImages(img)
|
||||
i = i + 1
|
||||
log.progress((i / total))
|
||||
|
||||
|
||||
json_input = json.loads(sys.stdin.read())
|
||||
|
||||
FRAGMENT_SERVER = json_input["server_connection"]
|
||||
stash = StashInterface(FRAGMENT_SERVER)
|
||||
|
||||
config = stash.get_configuration()["plugins"]
|
||||
settings = {
|
||||
"path": "/download_dir",
|
||||
"runPerformerScraper": False,
|
||||
}
|
||||
if "stashdb-performer-gallery" in config:
|
||||
settings.update(config["stashdb-performer-gallery"])
|
||||
# log.info('config: %s ' % (settings,))
|
||||
|
||||
|
||||
tag_stashbox_performer_gallery = stash.find_tag(
|
||||
"[Stashbox Performer Gallery]", create=True
|
||||
).get("id")
|
||||
tag_performer_image = stash.find_tag("[Set Profile Image]", create=True).get("id")
|
||||
|
||||
if "stasdb-performer-gallery" in config:
|
||||
settings.update(config["stasdb-performer-gallery"])
|
||||
|
||||
if "mode" in json_input["args"]:
|
||||
PLUGIN_ARGS = json_input["args"]["mode"]
|
||||
if "performer" in json_input["args"]:
|
||||
p = stash.find_performer(json_input["args"]["performer"])
|
||||
if tag_stashbox_performer_gallery in [x["id"] for x in p["tags"]]:
|
||||
processPerformer(p)
|
||||
stash.metadata_scan(paths=[settings["path"]])
|
||||
stash.run_plugin_task(
|
||||
"stashdb-performer-gallery",
|
||||
"relink missing images",
|
||||
args={"performer_id": p["id"]},
|
||||
)
|
||||
elif "processPerformers" in PLUGIN_ARGS:
|
||||
processPerformers()
|
||||
stash.metadata_scan([settings["path"]])
|
||||
stash.run_plugin_task(
|
||||
"stashdb-performer-gallery", "relink missing images", args={}
|
||||
)
|
||||
elif "processImages" in PLUGIN_ARGS:
|
||||
if "performer_id" in json_input["args"]:
|
||||
relink_images(performer_id=json_input["args"]["performer_id"])
|
||||
else:
|
||||
relink_images()
|
||||
|
||||
|
||||
elif "hookContext" in json_input["args"]:
|
||||
id = json_input["args"]["hookContext"]["id"]
|
||||
if json_input["args"]["hookContext"]["type"] == "Image.Create.Post":
|
||||
img = stash.find_image(image_in=id)
|
||||
processImages(img)
|
||||
if json_input["args"]["hookContext"]["type"] == "Image.Update.Post":
|
||||
img = stash.find_image(image_in=id)
|
||||
if tag_performer_image in [x["id"] for x in img["tags"]]:
|
||||
setPerformerPicture(img)
|
||||
if json_input["args"]["hookContext"]["type"] == "Performer.Update.Post":
|
||||
stash.run_plugin_task(
|
||||
"stashdb-performer-gallery", "Process Performers", args={"performer": id}
|
||||
)
|
||||
Reference in New Issue
Block a user