full KG from HF + simple Graph Search

This commit is contained in:
DaniG2106 2024-07-24 12:16:37 +02:00
parent d447b8f3ea
commit a295bc6657
10 changed files with 436480 additions and 160 deletions

BIN
.DS_Store vendored Normal file

Binary file not shown.

View File

@ -1,189 +1,259 @@
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix sc: <http://purl.org/science/owl/sciencecommons/> .
@prefix conn: <http://example.org/conn/> .
@prefix library: <http://example.org/library/> .
@prefix problem: <http://example.org/problem/> .
@prefix tag: <http://example.org/tag/> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
sc:Datatype a rdfs:Class .
<http://example.org/model/ActivationAI/distilbert-base-uncased-finetuned-emotion> conn:downloads 2926095 ;
conn:hasCoverTag tag:nlp ;
conn:hasProblem problem:text-classification ;
conn:hasTag tag:autotrain_compatible,
tag:distilbert,
tag:endpoints_compatible,
tag:generated_from_trainer,
tag:license%3Aapache-2.0,
tag:model-index,
tag:pytorch,
tag:region%3Aus,
tag:tensorboard,
tag:text-classification,
tag:transformers ;
conn:lastModified "2022-03-02T03:40:08+00:00"^^xsd:dateTime ;
conn:likes 2926095 ;
conn:model_id "621ffdc036468d709f1744de"^^xsd:string ;
conn:model_name "ActivationAI/distilbert-base-uncased-finetuned-emotion"^^xsd:string ;
conn:type conn:Model ;
conn:usesLibrary library:transformers .
sc:MLGoal a rdfs:Class .
<http://example.org/model/Necent/distilbert-base-uncased-detected-jailbreak> conn:downloads 4410904 ;
conn:hasCoverTag tag:nlp ;
conn:hasProblem problem:text-classification ;
conn:hasTag tag:autotrain_compatible,
tag:distilbert,
tag:endpoints_compatible,
tag:region%3Aus,
tag:safetensors,
tag:text-classification,
tag:transformers ;
conn:lastModified "2024-04-18T09:51:24+00:00"^^xsd:dateTime ;
conn:likes 4410904 ;
conn:model_id "661fd239da0148fbd982383e"^^xsd:string ;
conn:model_name "Necent/distilbert-base-uncased-detected-jailbreak"^^xsd:string ;
conn:type conn:Model ;
conn:usesLibrary library:transformers .
sc:Model a rdfs:Class .
<http://example.org/model/SamLowe/roberta-base-go_emotions> conn:downloads 2855930 ;
conn:hasCoverTag tag:nlp ;
conn:hasProblem problem:text-classification ;
conn:hasTag tag:autotrain_compatible,
tag:endpoints_compatible,
tag:license%3Amit,
tag:pytorch,
tag:region%3Aus,
tag:roberta,
tag:safetensors,
tag:text-classification,
tag:transformers ;
conn:lastModified "2023-10-04T10:00:58+00:00"^^xsd:dateTime ;
conn:likes 2855930 ;
conn:model_id "632322d59e488d65bf2f4742"^^xsd:string ;
conn:model_name "SamLowe/roberta-base-go_emotions"^^xsd:string ;
conn:type conn:Model ;
conn:usesLibrary library:transformers .
sc:Operation a rdfs:Class .
<http://example.org/model/avichr/heBERT_sentiment_analysis> conn:downloads 3710303 ;
conn:hasCoverTag tag:nlp ;
conn:hasProblem problem:text-classification ;
conn:hasTag tag:arxiv%3A1810.04805,
tag:autotrain_compatible,
tag:bert,
tag:endpoints_compatible,
tag:jax,
tag:pytorch,
tag:region%3Aus,
tag:text-classification,
tag:transformers ;
conn:lastModified "2021-12-31T16:08:22+00:00"^^xsd:dateTime ;
conn:likes 3710303 ;
conn:model_id "621ffdc136468d709f1796d3"^^xsd:string ;
conn:model_name "avichr/heBERT_sentiment_analysis"^^xsd:string ;
conn:type conn:Model ;
conn:usesLibrary library:transformers .
sc:AudioData a sc:TemporalData .
<http://example.org/model/cardiffnlp/twitter-roberta-base-sentiment-latest> conn:downloads 5338483 ;
conn:hasCoverTag tag:nlp ;
conn:hasProblem problem:text-classification ;
conn:hasTag tag:autotrain_compatible,
tag:endpoints_compatible,
tag:pytorch,
tag:region%3Aus,
tag:roberta,
tag:text-classification,
tag:transformers ;
conn:lastModified "2023-05-28T05:45:10+00:00"^^xsd:dateTime ;
conn:likes 5338483 ;
conn:model_id "622fea36174feb5439c2e4be"^^xsd:string ;
conn:model_name "cardiffnlp/twitter-roberta-base-sentiment-latest"^^xsd:string ;
conn:type conn:Model ;
conn:usesLibrary library:transformers .
sc:BERT a sc:Transformer ;
sc:hasParameters sc:167360261 ;
sc:input sc:TextData ;
sc:mlgoal sc:NaturalLanguageUnderstanding,
sc:SentimentAnalysis ;
sc:output sc:Labels .
<http://example.org/model/distilbert/distilbert-base-uncased-finetuned-sst-2-english> conn:downloads 7256764 ;
conn:hasCoverTag tag:nlp ;
conn:hasProblem problem:text-classification ;
conn:hasTag tag:autotrain_compatible,
tag:distilbert,
tag:endpoints_compatible,
tag:license%3Aapache-2.0,
tag:model-index,
tag:onnx,
tag:pytorch,
tag:region%3Aus,
tag:rust,
tag:safetensors,
tag:text-classification,
tag:transformers ;
conn:lastModified "2023-12-19T16:29:37+00:00"^^xsd:dateTime ;
conn:likes 7256764 ;
conn:model_id "621ffdc036468d709f174346"^^xsd:string ;
conn:model_name "distilbert/distilbert-base-uncased-finetuned-sst-2-english"^^xsd:string ;
conn:type conn:Model ;
conn:usesLibrary library:transformers .
sc:Cropping a sc:Operation ;
sc:input sc:BoundingBox,
sc:ImageData ;
sc:output sc:ImageData .
<http://example.org/model/facebook/bart-large-mnli> conn:downloads 2929233 ;
conn:hasCoverTag tag:nlp ;
conn:hasProblem problem:text-classification ;
conn:hasTag tag:autotrain_compatible,
tag:bart,
tag:dataset%3Amulti_nli,
tag:endpoints_compatible,
tag:jax,
tag:license%3Amit,
tag:pytorch,
tag:region%3Aus,
tag:rust,
tag:safetensors,
tag:text-classification,
tag:transformers,
tag:zero-shot-classification ;
conn:lastModified "2023-09-05T14:49:34+00:00"^^xsd:dateTime ;
conn:likes 2929233 ;
conn:model_id "621ffdc136468d709f17adb7"^^xsd:string ;
conn:model_name "facebook/bart-large-mnli"^^xsd:string ;
conn:type conn:Model ;
conn:usesLibrary library:transformers .
sc:DETA a sc:Transformer ;
sc:hasParameters sc:48438489 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ObjectDetection ;
sc:output sc:BoundingBox,
sc:Labels .
<http://example.org/model/facebook/fasttext-language-identification> conn:downloads 53107820 ;
conn:hasCoverTag tag:nlp ;
conn:hasProblem problem:text-classification ;
conn:hasTag tag:license%3Acc-by-nc-4.0,
tag:region%3Aus,
tag:text-classification ;
conn:lastModified "2023-06-09T12:39:43+00:00"^^xsd:dateTime ;
conn:likes 53107820 ;
conn:model_id "6405e2229f5bb41f15e94b68"^^xsd:string ;
conn:model_name "facebook/fasttext-language-identification"^^xsd:string ;
conn:type conn:Model ;
conn:usesLibrary library:fasttext .
sc:DETR a sc:Transformer ;
sc:hasParameters sc:41524768 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ObjectDetection ;
sc:output sc:BoundingBox,
sc:Labels .
<http://example.org/model/mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis> conn:downloads 3924929 ;
conn:hasCoverTag tag:nlp ;
conn:hasProblem problem:text-classification ;
conn:hasTag tag:autotrain_compatible,
tag:endpoints_compatible,
tag:generated_from_trainer,
tag:license%3Aapache-2.0,
tag:model-index,
tag:pytorch,
tag:region%3Aus,
tag:roberta,
tag:safetensors,
tag:tensorboard,
tag:text-classification,
tag:transformers ;
conn:lastModified "2024-01-21T15:17:58+00:00"^^xsd:dateTime ;
conn:likes 3924929 ;
conn:model_id "621ffdc136468d709f17e2c1"^^xsd:string ;
conn:model_name "mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis"^^xsd:string ;
conn:type conn:Model ;
conn:usesLibrary library:transformers .
sc:EEGData a sc:TemporalData .
<http://example.org/model/papluca/xlm-roberta-base-language-detection> conn:downloads 3107915 ;
conn:hasCoverTag tag:nlp ;
conn:hasProblem problem:text-classification ;
conn:hasTag tag:autotrain_compatible,
tag:endpoints_compatible,
tag:generated_from_trainer,
tag:license%3Amit,
tag:multilingual,
tag:pytorch,
tag:region%3Aus,
tag:safetensors,
tag:text-classification,
tag:transformers,
tag:xlm-roberta ;
conn:lastModified "2023-12-28T13:54:18+00:00"^^xsd:dateTime ;
conn:likes 3107915 ;
conn:model_id "621ffdc136468d709f17ebd6"^^xsd:string ;
conn:model_name "papluca/xlm-roberta-base-language-detection"^^xsd:string ;
conn:type conn:Model ;
conn:usesLibrary library:transformers .
sc:FullyConnected a sc:Model ;
sc:input sc:Features ;
sc:mlgoal sc:Classification ;
sc:output sc:Labels .
library:fasttext conn:type conn:Library .
sc:IMUData a sc:TemporalData .
tag:arxiv%3A1810.04805 conn:type conn:Tag .
sc:InceptionV4 a sc:Model ;
sc:hasParameters sc:42679816 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ImageClassification ;
sc:output sc:Labels .
tag:bart conn:type conn:Tag .
sc:OneFormer a sc:Transformer ;
sc:hasParameters sc:218781179 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ImageSegmentation ;
sc:output sc:Labels .
tag:bert conn:type conn:Tag .
sc:Open13 a sc:Model ;
sc:input sc:Spectrogram ;
sc:mlgoal sc:InformationEmbedding ;
sc:output sc:Features .
tag:dataset%3Amulti_nli conn:type conn:Tag .
sc:SAM a sc:Transformer ;
sc:hasParameters sc:93735472 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ImageSegmentation ;
sc:output sc:Labels .
tag:license%3Acc-by-nc-4.0 conn:type conn:Tag .
sc:STFT a sc:Operation ;
sc:input sc:TemporalData ;
sc:output sc:Spectrogram .
tag:multilingual conn:type conn:Tag .
sc:TemporalConvolution a sc:Model ;
sc:input sc:TemporalData ;
sc:mlgoal sc:InformationEmbedding ;
sc:output sc:Features .
tag:onnx conn:type conn:Tag .
sc:VIT a sc:Model ;
sc:hasParameters sc:86567656 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ImageClassification ;
sc:output sc:Labels .
tag:xlm-roberta conn:type conn:Tag .
sc:VideoData a sc:CompositeData ;
sc:TemporalSeq sc:ImageData ;
sc:is sc:SpatialData,
sc:TemporalData .
tag:zero-shot-classification conn:type conn:Tag .
sc:YOLO a sc:Model ;
sc:hasParameters sc:68229648 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ObjectDetection ;
sc:output sc:BoundingBox,
sc:Labels .
tag:jax conn:type conn:Tag .
sc:YOLOS a sc:Transformer ;
sc:hasParameters sc:6488736 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ObjectDetection ;
sc:output sc:BoundingBox,
sc:Labels .
tag:rust conn:type conn:Tag .
sc:distilbert a sc:Transformer ;
sc:hasParameters sc:135326979 ;
sc:input sc:TextData ;
sc:mlgoal sc:NaturalLanguageUnderstanding,
sc:SentimentAnalysis ;
sc:output sc:Labels .
tag:tensorboard conn:type conn:Tag .
sc:135326979 a sc:numberOfParameters .
tag:distilbert conn:type conn:Tag .
sc:167360261 a sc:numberOfParameters .
tag:generated_from_trainer conn:type conn:Tag .
sc:218781179 a sc:numberOfParameters .
tag:license%3Aapache-2.0 conn:type conn:Tag .
sc:41524768 a sc:numberOfParameters .
tag:license%3Amit conn:type conn:Tag .
sc:42679816 a sc:numberOfParameters .
tag:model-index conn:type conn:Tag .
sc:48438489 a sc:numberOfParameters .
tag:roberta conn:type conn:Tag .
sc:6488736 a sc:numberOfParameters .
tag:safetensors conn:type conn:Tag .
sc:68229648 a sc:numberOfParameters .
tag:pytorch conn:type conn:Tag .
sc:86567656 a sc:numberOfParameters .
library:transformers conn:type conn:Library .
sc:93735472 a sc:numberOfParameters .
tag:autotrain_compatible conn:type conn:Tag .
sc:SpectralDensity a sc:DiscreteData .
tag:endpoints_compatible conn:type conn:Tag .
sc:CompositeData a sc:Datatype .
tag:transformers conn:type conn:Tag .
sc:ContinuousData a sc:Datatype .
problem:text-classification conn:isA tag:nlp ;
conn:type conn:Problem .
sc:ImageClassification a sc:Classification ;
sc:input sc:ImageData ;
sc:output sc:Labels .
tag:region%3Aus conn:type conn:Tag .
sc:ImageSegmentation a sc:Classification ;
sc:input sc:ImageData ;
sc:output sc:Labels .
tag:text-classification conn:type conn:Tag .
sc:InformationEmbedding a sc:MLGoal ;
sc:output sc:Features .
sc:SpatialData a sc:ContinuousData .
sc:Spectrogram a sc:CompositeData ;
sc:TemporalSeq sc:SpectralDensity ;
sc:islike sc:ImageData .
sc:Classification a sc:MLGoal ;
sc:input sc:Features ;
sc:output sc:Labels .
sc:NaturalLanguageUnderstanding a sc:MLGoal ;
sc:input sc:TextData ;
sc:output sc:Features .
sc:SentimentAnalysis a sc:NaturalLanguageUnderstanding ;
sc:output sc:Labels .
sc:TextData a sc:DiscreteData ;
sc:input sc:SentimentAnalysis .
sc:ObjectDetection a sc:MLGoal ;
sc:input sc:ImageData ;
sc:output sc:BoundingBox .
sc:DiscreteData a sc:Datatype .
sc:BoundingBox a sc:DiscreteData .
sc:Features a sc:DiscreteData .
sc:TemporalData a sc:ContinuousData .
sc:Transformer a sc:Model .
sc:numberOfParameters a rdfs:Class .
sc:ImageData a sc:SpatialData .
sc:Labels a sc:DiscreteData .
tag:nlp conn:type conn:CoverTag .

View File

@ -0,0 +1,189 @@
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix sc: <http://purl.org/science/owl/sciencecommons/> .
sc:Datatype a rdfs:Class .
sc:MLGoal a rdfs:Class .
sc:Model a rdfs:Class .
sc:Operation a rdfs:Class .
sc:AudioData a sc:TemporalData .
sc:BERT a sc:Transformer ;
sc:hasParameters sc:167360261 ;
sc:input sc:TextData ;
sc:mlgoal sc:NaturalLanguageUnderstanding,
sc:SentimentAnalysis ;
sc:output sc:Labels .
sc:Cropping a sc:Operation ;
sc:input sc:BoundingBox,
sc:ImageData ;
sc:output sc:ImageData .
sc:DETA a sc:Transformer ;
sc:hasParameters sc:48438489 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ObjectDetection ;
sc:output sc:BoundingBox,
sc:Labels .
sc:DETR a sc:Transformer ;
sc:hasParameters sc:41524768 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ObjectDetection ;
sc:output sc:BoundingBox,
sc:Labels .
sc:EEGData a sc:TemporalData .
sc:FullyConnected a sc:Model ;
sc:input sc:Features ;
sc:mlgoal sc:Classification ;
sc:output sc:Labels .
sc:IMUData a sc:TemporalData .
sc:InceptionV4 a sc:Model ;
sc:hasParameters sc:42679816 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ImageClassification ;
sc:output sc:Labels .
sc:OneFormer a sc:Transformer ;
sc:hasParameters sc:218781179 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ImageSegmentation ;
sc:output sc:Labels .
sc:Open13 a sc:Model ;
sc:input sc:Spectrogram ;
sc:mlgoal sc:InformationEmbedding ;
sc:output sc:Features .
sc:SAM a sc:Transformer ;
sc:hasParameters sc:93735472 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ImageSegmentation ;
sc:output sc:Labels .
sc:STFT a sc:Operation ;
sc:input sc:TemporalData ;
sc:output sc:Spectrogram .
sc:TemporalConvolution a sc:Model ;
sc:input sc:TemporalData ;
sc:mlgoal sc:InformationEmbedding ;
sc:output sc:Features .
sc:VIT a sc:Model ;
sc:hasParameters sc:86567656 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ImageClassification ;
sc:output sc:Labels .
sc:VideoData a sc:CompositeData ;
sc:TemporalSeq sc:ImageData ;
sc:is sc:SpatialData,
sc:TemporalData .
sc:YOLO a sc:Model ;
sc:hasParameters sc:68229648 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ObjectDetection ;
sc:output sc:BoundingBox,
sc:Labels .
sc:YOLOS a sc:Transformer ;
sc:hasParameters sc:6488736 ;
sc:input sc:ImageData ;
sc:mlgoal sc:ObjectDetection ;
sc:output sc:BoundingBox,
sc:Labels .
sc:distilbert a sc:Transformer ;
sc:hasParameters sc:135326979 ;
sc:input sc:TextData ;
sc:mlgoal sc:NaturalLanguageUnderstanding,
sc:SentimentAnalysis ;
sc:output sc:Labels .
sc:135326979 a sc:numberOfParameters .
sc:167360261 a sc:numberOfParameters .
sc:218781179 a sc:numberOfParameters .
sc:41524768 a sc:numberOfParameters .
sc:42679816 a sc:numberOfParameters .
sc:48438489 a sc:numberOfParameters .
sc:6488736 a sc:numberOfParameters .
sc:68229648 a sc:numberOfParameters .
sc:86567656 a sc:numberOfParameters .
sc:93735472 a sc:numberOfParameters .
sc:SpectralDensity a sc:DiscreteData .
sc:CompositeData a sc:Datatype .
sc:ContinuousData a sc:Datatype .
sc:ImageClassification a sc:Classification ;
sc:input sc:ImageData ;
sc:output sc:Labels .
sc:ImageSegmentation a sc:Classification ;
sc:input sc:ImageData ;
sc:output sc:Labels .
sc:InformationEmbedding a sc:MLGoal ;
sc:output sc:Features .
sc:SpatialData a sc:ContinuousData .
sc:Spectrogram a sc:CompositeData ;
sc:TemporalSeq sc:SpectralDensity ;
sc:islike sc:ImageData .
sc:Classification a sc:MLGoal ;
sc:input sc:Features ;
sc:output sc:Labels .
sc:NaturalLanguageUnderstanding a sc:MLGoal ;
sc:input sc:TextData ;
sc:output sc:Features .
sc:SentimentAnalysis a sc:NaturalLanguageUnderstanding ;
sc:output sc:Labels .
sc:TextData a sc:DiscreteData ;
sc:input sc:SentimentAnalysis .
sc:ObjectDetection a sc:MLGoal ;
sc:input sc:ImageData ;
sc:output sc:BoundingBox .
sc:DiscreteData a sc:Datatype .
sc:BoundingBox a sc:DiscreteData .
sc:Features a sc:DiscreteData .
sc:TemporalData a sc:ContinuousData .
sc:Transformer a sc:Model .
sc:numberOfParameters a rdfs:Class .
sc:ImageData a sc:SpatialData .
sc:Labels a sc:DiscreteData .

View File

@ -13,7 +13,7 @@ tags = json.loads(response.content)["pipeline_tag"]
print(tags)
conn = sqlite3.connect('huggingface_data.db')
conn = sqlite3.connect('huggingface.db')
cursor = conn.cursor()
@ -21,9 +21,9 @@ cursor.execute('''
CREATE TABLE IF NOT EXISTS Models (
model_id TEXT PRIMARY KEY,
model_name TEXT,
tag TEXT,
customTags TEXT,
subType TEXT,
problem TEXT,
tags TEXT,
coverTag TEXT,
library TEXT,
downloads INTEGER,
likes INTEGER,
@ -33,14 +33,14 @@ CREATE TABLE IF NOT EXISTS Models (
conn.commit()
for tag in tags:
tag_name = tag["id"]
subType = tag["subType"]
problem = tag["id"]
coverTag = tag["subType"]
print(tag_name)
print(problem)
response = requests.get(
"https://huggingface.co/api/models",
params={"limit": "unlimited", "full": "True", "config": "True", "sort": "downloads", "filter": tag_name},
params={"limit": "unlimited", "full": "True", "config": "True", "sort": "downloads", "filter": problem},
headers={}
)
data = json.loads(response.content)
@ -49,16 +49,16 @@ for tag in tags:
try:
model_id = model['_id']
model_name = model['id']
customTags = json.dumps(model['tags'])
tags = json.dumps(model['tags'])
library = model["library_name"]
downloads = model['downloads']
likes = model['downloads']
modified = model['lastModified']
cursor.execute('''
INSERT INTO Models (model_id, model_name, tag, customTags, subType, library, downloads, likes, lastModified)
INSERT INTO Models (model_id, model_name, problem, tags, coverTag, library, downloads, likes, lastModified)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (model_id, model_name, tag_name, customTags, subType, library, downloads, likes, modified))
''', (model_id, model_name, problem, tags, coverTag, library, downloads, likes, modified))
except sqlite3.IntegrityError:
# print(model)
pass

435508
Hugging2Graph/graph.ttl Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,153 @@
from rdflib import Graph, Namespace, RDF, Literal
from rdflib.namespace import XSD
def load_graph(file_path):
g = Graph()
g.parse(file_path, format="turtle")
return g
def get_cover_tags(graph):
query = """
PREFIX conn: <http://example.org/conn/>
SELECT ?coverTag
WHERE {
?coverTag a conn:CoverTag .
}
"""
results = graph.query(query)
cover_tags = [row[0] for row in results]
return cover_tags
def get_problems(graph):
query = """
PREFIX conn: <http://example.org/conn/>
SELECT ?problem
WHERE {
?problem a conn:Problem .
}
"""
results = graph.query(query)
problems = [row[0] for row in results]
return problems
def get_problems_for_cover_tag(graph, cover_tag_literal_text):
cover_tag_literal = Literal(cover_tag_literal_text, datatype=XSD.string)
query = """
PREFIX conn: <http://example.org/conn/>
PREFIX problem: <http://example.org/problem/>
PREFIX tag: <http://example.org/tag/>
SELECT ?problem
WHERE {
?problem a conn:Problem .
?problem conn:hasCoverTag ?coverTag .
FILTER (?coverTag = ?cover_tag_literal)
}
"""
results = graph.query(query, initBindings={'cover_tag_literal': cover_tag_literal})
problems = [row[0] for row in results]
return problems
def get_models_for_problem(graph, problem_literal_text):
problem_literal = Literal(problem_literal_text, datatype=XSD.string)
query = """
PREFIX conn: <http://example.org/conn/>
PREFIX model: <http://example.org/model/>
SELECT ?model ?downloads
WHERE {
?model a conn:Model .
?model conn:hasProblem ?problem .
?model conn:downloads ?downloads .
FILTER (?problem = ?problem_literal)
}
ORDER BY DESC(?downloads)
"""
results = graph.query(query, initBindings={'problem_literal': problem_literal})
models = [(row[0], row[1]) for row in results]
return models
def get_model_details(graph, model_name):
model_literal = Literal(model_name, datatype=XSD.string)
query = """
PREFIX conn: <http://example.org/conn/>
PREFIX model: <http://example.org/model/>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
SELECT ?model ?id ?name ?problem ?coverTag ?library ?downloads ?likes ?lastModified
WHERE {
?model a conn:Model .
?model conn:model_name ?name .
?model conn:model_id ?id .
?model conn:hasProblem ?problem .
?model conn:hasCoverTag ?coverTag .
?model conn:usesLibrary ?library .
?model conn:downloads ?downloads .
?model conn:likes ?likes .
?model conn:lastModified ?lastModified .
FILTER (?name = ?model_literal)
}
"""
results = graph.query(query, initBindings={'model_literal': model_literal})
details = {}
for row in results:
details = {
'model_uri': row[0],
'id': row[1],
'name': row[2],
'problem': row[3],
'coverTag': row[4],
'library': row[5],
'downloads': row[6],
'likes': row[7],
'lastModified': row[8]
}
return details
def print_results(literals, label):
print(f"List of available {label}:")
for literal in literals:
print(literal)
print()
def print_models(models):
print("Models ordered by downloads:")
for model, downloads in models:
print(f"Model: {model}, Downloads: {downloads}")
print()
def print_model_details(details):
print("Model details:")
for key, value in details.items():
print(f"{key}: {value}")
print()
if __name__ == "__main__":
graph = load_graph("graph.ttl")
# Get Modality (Cover Tags)
cover_tags = get_cover_tags(graph)
print_results(cover_tags, "cover tags")
# Get Problem List
problems = get_problems(graph)
print_results(problems, "problems")
cover_tag = "audio"
problems = get_problems_for_cover_tag(graph, cover_tag)
print_results(problems, "problems connected to the cover tag")
# Get Model by Problem to solve
problem_literal_text = "image-to-text"
models = get_models_for_problem(graph, problem_literal_text)
print_models(models)
# Get Model Details (Example)
model_name = "OleehyO/TexTeller"
model_details = get_model_details(graph, model_name)
print_model_details(model_details)

View File

@ -0,0 +1,89 @@
import sqlite3
from rdflib import Graph, Literal, Namespace, RDF
from rdflib.namespace import XSD
import ast
# Namespaces
CONN = Namespace("http://example.org/conn/")
MODEL = Namespace("http://example.org/model/")
PROBLEM = Namespace("http://example.org/problem/")
TAG = Namespace("http://example.org/tag/")
LIBRARY = Namespace("http://example.org/library/")
# Extract Tags from Stringified List
allowed_tags = set()
with open('topTags.txt', 'r', encoding='utf-8') as file:
for line in file:
tag = line.strip()
if tag:
allowed_tags.add(tag)
conn = sqlite3.connect('huggingface.db')
cursor = conn.cursor()
cursor.execute("SELECT * FROM Models")
rows = cursor.fetchall()
g = Graph()
# Namespaces
g.bind("conn", CONN)
g.bind("model", MODEL)
g.bind("problem", PROBLEM)
g.bind("tag", TAG)
g.bind("library", LIBRARY)
# Add triples to the graph
for row in rows:
model_id = row[0]
model_name = row[1]
problem = row[2]
tags = row[3]
coverTag = row[4]
library = row[5]
downloads = row[6]
likes = row[7]
lastModified = row[8]
# Literals for string based KG
model_node = Literal(model_name)
problem_node = Literal(problem)
coverTag_node = Literal(coverTag)
library_node = Literal(library)
# Add Types
g.add((model_node, RDF.type, CONN.Model))
g.add((problem_node, RDF.type, CONN.Problem))
g.add((coverTag_node, RDF.type, CONN.CoverTag))
g.add((library_node, RDF.type, CONN.Library))
# Add Model Details
g.add((model_node, CONN.model_name, Literal(model_name, datatype=XSD.string)))
g.add((model_node, CONN.model_id, Literal(model_id, datatype=XSD.string)))
g.add((model_node, CONN.downloads, Literal(downloads, datatype=XSD.integer)))
g.add((model_node, CONN.likes, Literal(likes, datatype=XSD.integer)))
g.add((model_node, CONN.lastModified, Literal(lastModified, datatype=XSD.dateTime)))
# Relationships
g.add((model_node, CONN.hasProblem, problem_node))
g.add((model_node, CONN.hasCoverTag, coverTag_node))
g.add((model_node, CONN.usesLibrary, library_node))
g.add((problem_node, CONN.hasCoverTag, coverTag_node))
try:
tags_list = ast.literal_eval(tags)
if isinstance(tags_list, list):
for tag in tags_list:
tag = tag.strip()
if tag in allowed_tags:
tag_literal = Literal(tag, datatype=XSD.string)
g.add((model_node, CONN.hasTag, tag_literal))
g.add((tag_literal, RDF.type, CONN.Tag))
else:
print(f"Tags for model {model_name} are not in a list format")
except Exception as e:
print(f"Error parsing tags for model {model_name}: {e}")
# TO FILE
g.serialize("graph.ttl", format="turtle")
conn.close()

View File

@ -0,0 +1,44 @@
import sqlite3
import ast
from collections import Counter
conn = sqlite3.connect('huggingface.db')
cursor = conn.cursor()
cursor.execute('SELECT tags FROM Models')
rows = cursor.fetchall()
tag_counter = Counter()
for id,row in enumerate(rows):
try:
tags_str = row[0]
tags = ast.literal_eval(tags_str)
if isinstance(tags, list):
tag_counter.update(tags)
else:
print(f"Unexpected format, not a list: {tags_str}")
except (ValueError, SyntaxError, UnicodeEncodeError) as e:
# Some Strings are broken from Huggingface
print(f"Error encountered, skipping row: {e}")
print(row)
conn.close()
ranked_tags = tag_counter.most_common()
def safe_format(tag, count):
try:
# return f"{tag}; {count}".encode('utf-8', errors='replace').decode('utf-8')
return f"{tag}".encode('utf-8', errors='replace').decode('utf-8')
except UnicodeEncodeError:
return f"Encoding error for tag {tag}: {count}"
output_file = 'topTags.txt'
occurence = 50
with open(output_file, 'w', encoding='utf-8') as file:
for tag, count in ranked_tags:
if count >=occurence and len(tag)>2:
file.write(safe_format(tag, count) + '\n')

267
Hugging2Graph/topTags.txt Normal file
View File

@ -0,0 +1,267 @@
region:us
endpoints_compatible
transformers
pytorch
safetensors
autotrain_compatible
license:apache-2.0
tensorboard
license:mit
generated_from_trainer
text2text-generation
diffusers
bert
model-index
text-generation-inference
feature-extraction
onnx
license:cc-by-nc-4.0
text-generation
text-classification
vision
text-to-audio
jax
text-to-image
license:other
sentence-similarity
fill-mask
image-classification
automatic-speech-recognition
token-classification
object-detection
reinforcement-learning
audio-classification
unconditional-image-generation
translation
audio-to-audio
text-to-speech
question-answering
summarization
video-classification
stable-diffusion
deep-reinforcement-learning
sentence-transformers
diffusers:DDPMPipeline
diffusion-models-class
ml-agents
text-embeddings-inference
rvc
sail-rvc
roberta
videomae
wav2vec2
marian
timm
arxiv:1910.09700
image-segmentation
audio
dataset:imagenet-1k
license:creativeml-openrail-m
diffusers:StableDiffusionPipeline
custom_code
vits
image-to-text
detr
ML-Agents-Huggy
xlm-roberta
speecht5
zero-shot-image-classification
stable-diffusion-diffusers
llama
whisper
mms
arxiv:2305.13516
multilingual
license:cc-by-4.0
segformer
clip
image-text-to-text
zero-shot-classification
unity-ml-agents
visual-question-answering
distilbert
image-to-image
conversational
bart
Huggy
hubert
transformers.js
speech
image-feature-extraction
base_model:MCG-NJU/videomae-base
base_model:microsoft/speecht5_tts
mteb
gguf
arxiv:2010.11929
tabular-classification
diffusers:StableDiffusionXLPipeline
dataset:marsyas/gtzan
co2_eq_emissions
vision-encoder-decoder
hf-asr-leaderboard
deberta-v2
license:bsd-3-clause
autotrain
arxiv:1908.10084
pretraining
mt5
art
stable-diffusion-xl
license:cc-by-sa-4.0
license:cc-by-nc-sa-4.0
mistral
open_clip
document-question-answering
image-captioning
license:openrail
license:gemma
ML-Agents-SnowballTarget
audio-spectrogram-transformer
dataset:wikipedia
vit
dataset:facebook/voxpopuli
sklearn
espnet
ML-Agents-SoccerTwos
arxiv:2303.15343
layoutlmv2
base_model:stabilityai/stable-diffusion-xl-base-1.0
code
arxiv:1908.04913
arxiv:1804.00015
paligemma
medical
lora
anime
arxiv:2209.06794
dataset:squad
mask-generation
dataset:common_voice
yolo
4-bit
base_model:nvidia/mit-b0
SnowballTarget
sam
blip
table-transformer
arxiv:1905.13648
arxiv:1706.03762
arxiv:2103.01913
arxiv:2310.09199
arxiv:2403.08295
arxiv:2209.04372
arxiv:2205.12522
arxiv:2110.11624
arxiv:2108.03353
arxiv:2010.04295
arxiv:2401.06209
arxiv:2305.10355
arxiv:2203.10244
arxiv:1810.12440
arxiv:1608.00272
joblib
arxiv:2407.07726
ceb
biology
dataset:mozilla-foundation/common_voice_11_0
llava
controlnet
electra
gpt2
yolos
albert
skops
whisper-event
dataset:voxpopuli
rust
vilt
depth-estimation
arxiv:1512.03385
camembert
table-question-answering
text-to-video
tabular-regression
base_model:ntu-spml/distilhubert
license:gpl-3.0
blip-2
haw
mpnet
dataset:imagenet-22k
license:openrail++
generated_from_keras_callback
dataset:imagenet-21k
ultralytics
xlsr-fine-tuning-week
yolov8
ultra-realistic
dataset:squad_v2
ultralyticsplus
base_model:facebook/detr-resnet-50
stable-diffusion-api
arxiv:2110.00476
adapter-transformers
mbart
arxiv:2103.00020
arxiv:2201.03545
license:llama2
dataset:superb
fairseq
vivit
dataset:multi_nli
classification
tabular
deberta
base_model:finetune:microsoft/speecht5_tts
pony
tf-keras
dataset:coco
base_model:runwayml/stable-diffusion-v1-5
arxiv:1705.00652
peft
loss:MultipleNegativesRankingLoss
Anime
bitsandbytes
unk
ner
PyTorch
base_model:finetune:MCG-NJU/videomae-base
license:llama3
pegasus
robust-speech-event
template:sd-lora
stable-baselines3
arxiv:1910.10683
speechbrain
base_model:openai/whisper-small
timesformer
arxiv:1907.11692
Transformer
arxiv:1810.04805
license:unknown
nso
arxiv:2112.10752
spacy
qwen2
nemo
fon
dataset:imagefolder
awesome-yolov8-models
big_vision
llama-2
facebook
dataset:cnn_dailymail
ML-Agents-Pyramids
war
yue
sequence-tagger-model
flair
chemistry
quantized
dataset:Lin-Chen/ShareGPT4V
license:agpl-3.0
base_model:MCG-NJU/videomae-base-finetuned-kinetics
exbert
gpt_neox
hmn
longt5
arxiv:2302.05543