From 68c762debc4008f7ea03b6f46091c7d0830564f8 Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 14:36:30 +0000 Subject: [PATCH 01/21] Update corpora/florilege/labels.stats --- corpora/florilege/labels.stats | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/corpora/florilege/labels.stats b/corpora/florilege/labels.stats index b3c55895..2506702e 100644 --- a/corpora/florilege/labels.stats +++ b/corpora/florilege/labels.stats @@ -1,18 +1,21 @@ id,libelle,uri -cirm_000, date de mise à jour des données de cirm, /db/maj/genbank/date +cirm_000, date de mise à jour des données cirm, /db/maj/genbank/date cirm_001,nombre d'entrées de cirm-BIA,cirm/BIA_2021/florilege_export_final_17_02_21.xlsx -cirm_002,nombre d'entrées cirm-Levure,cirm/Levures_2021/Florilege_21012021.xlsx -cirm_003,nombre de taxons de cirm-BIA,cirm/mapped_taxids.txt +cirm_002,nombre d'entrées de cirm-Levure,cirm/Levures_2021/Florilege_21012021.xlsx +cirm_003,nombre de taxons de cirm-BIA,cirm/mapped_bia_taxa.txt cirm_004,nombre de taxons de cirm-Levure,cirm/mapped_yeast_taxa.txt cirm_005,nombre d'habitats de cirm-BIA,cirm/mapped_habitats.txt -cirm_006,nombre d'habitat de cirm-Levure,cirm/mapped_yeast_habitats.txt -genbank_000, date de mise à jour des données de genbank,/db/maj/genbank/date +cirm_006,nombre d'habitats de cirm-Levure,cirm/mapped_yeast_habitats.txt +cirm_007,nombre d'entrées de cirm-cfpb,corpora/cirm/CFBP_2020/CFPB_22_sept_2020_Type.xlsx +cirm_006,nombre de taxons de cirm-cfpb,cirm/mapped_cfbp_taxa.txt +cirm_006,nombre d'habitats de cirm-cfpb,cirm/mapped_cfbp_habitats.txt +genbank_000, date de mise à jour des données genbank,/db/maj/genbank/date genbank_001,nombre d'entrées de genbank,genbank/GenBank_extraction_20210127.tsv -genbank_002,nombre de taxon de genbank,genbank/mapped_taxids.txt +genbank_002,nombre de taxons de genbank,genbank/mapped_taxa.txt genbank_003,nombre d'entités du type #Habitat de genbank,genbank/mapped_habitats.txt dsmz_000, date de mise à jour des données de dsmz,/db/maj/dsmz/date dsmz_001,nombre d'entrées de dsmz,dsmz/dsmz-data/category=from_ncbi_taxonomy-key=taxid.tsv -dsmz_002,nombre de taxon venant de dsmz,dsmz/mapped_taxids.txt +dsmz_002,nombre de taxon venant de dsmz,dsmz/mapped_taxa.txt dsmz_003,nombre d'entités de type #Habitat de dsmz,dsmz/mapped_habitats.txt pubmed_000, date de mise à jour du corpus pubmed,/db/maj/pubmed/date pubmed_001,nombre de batches (x1000) pubmed,microbes-2019/list_of_batches.txt @@ -28,9 +31,9 @@ eval_001, corpus utilisés, https://sites.google.com/view/bb-2019 eval_002, date d'évaluation, migale/evaluation/BB19/date eval_BB19-norm+ner_001, mesure pour l'evaluation de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Mesure eval_BB19-norm+ner_002, score global sur la prédiction de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Standard_scoring -eval_BB19-norm+ner_002, score sur la prédiction des taxons de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Habitat +eval_BB19-norm+ner_002, score sur la prédiction des taxons de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Microorganism eval_BB19-norm+ner_003, score sur la prédiction des phénotypes de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Phenotype -eval_BB19-norm+ner_004, score sur la prédiction des habitats de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Microorganism +eval_BB19-norm+ner_004, score sur la prédiction des habitats de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Habitat eval_BB19-rel+ner_001, mesure pour l'evaluation de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Mesure eval_BB19-rel+ner_002, score global sur la prédiction de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Standard_scoring eval_BB19-rel+ner_002, score sur la prédiction des Lives-In de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Lives_In -- GitLab From 03d3b7c1f4c289fae6e00337e193e7c642171aaf Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 14:43:29 +0000 Subject: [PATCH 02/21] Update corpora/florilege/labels.stats Deleted corpora/florilege/stats.labels --- corpora/florilege/labels.stats | 11 +++++------ corpora/florilege/stats.labels | 22 ---------------------- 2 files changed, 5 insertions(+), 28 deletions(-) delete mode 100644 corpora/florilege/stats.labels diff --git a/corpora/florilege/labels.stats b/corpora/florilege/labels.stats index 2506702e..9082013d 100644 --- a/corpora/florilege/labels.stats +++ b/corpora/florilege/labels.stats @@ -21,12 +21,11 @@ pubmed_000, date de mise à jour du corpus pubmed,/db/maj/pubmed/date pubmed_001,nombre de batches (x1000) pubmed,microbes-2019/list_of_batches.txt pubmed_002,nombre d'entités du type #Habitat de pubmed,microbes-2019/habitats.full.txt pubmed_003,nombre d'entités du type #Taxon pubmed,microbes-2019/microorganisms.full.txt -pubmed_004,nombre de relations du type #Phenotype-Taxon pubmed,microbes-2019/phenotype-relations.full.txt -pubmed_005,nombre de relations du type #Phenotype-Relations pubmed,microbes-2019/phenotype-relations.txt -pubmed_006,nombre d'entités du type #Phenotype de pubmed,microbes-2019/phenotypes.full.txt -pubmed_007,nombre de relations du type #Taxon-Habitat de pubmed,microbes-2019/relations.full.txt -pubmed_008,nombre de relations de type #Use pubmed,microbes-2019/uses.full.txt -pubmed_009,nombre de relations du type #Use-Taxon pubmed,microbes-2019/uses-relations.full.txt +pubmed_004,nombre d'entités du type #Phenotype de pubmed,microbes-2019/phenotypes.full.txt +pubmed_005,nombre de relations de type #Use pubmed,microbes-2019/uses.full.txt +pubmed_006,nombre de relations du type #Taxon-Habitat de pubmed,microbes-2019/habitat-relations.full.txt +pubmed_007,nombre de relations du type #Phenotype-Taxon pubmed,microbes-2019/phenotype-relations.full.txt +pubmed_008,nombre de relations du type #Use-Taxon pubmed,microbes-2019/uses-relations.full.txt eval_001, corpus utilisés, https://sites.google.com/view/bb-2019 eval_002, date d'évaluation, migale/evaluation/BB19/date eval_BB19-norm+ner_001, mesure pour l'evaluation de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Mesure diff --git a/corpora/florilege/stats.labels b/corpora/florilege/stats.labels deleted file mode 100644 index 3be2a540..00000000 --- a/corpora/florilege/stats.labels +++ /dev/null @@ -1,22 +0,0 @@ -LIBELLE,file -entrées cirm,cirm/2019-07-05/extraction_3-fv.csv -entrées cirm (levure),cirm/Levures_2017/data_CIRM_levures_extraction_09032017.csv -taxid cirm,cirm/mapped_taxids.txt -yeast cirm,cirm/yeast_taxa.txt -habitats cirm,cirm/mapped_habitats.txt -habitats cirm (yeast),cirm/mapped_yeast_habitats.txt -entrées genbank,genbank/req1_sup800_bacteria-descriptors.csv -taxid genbank,genbank/mapped_taxids.txt -habitats genbank,genbank/mapped_habitats.txt -entrées dsmz,dsmz/dsmz-data/category=from_ncbi_taxonomy-key=taxid.tsv -taxid dsmz,dsmz/mapped_taxids.txt -habitats dsmz,dsmz/mapped_habitats.txt -batch (x1000) pubmed,microbes-2019/list_of_batches.txt -habitats pubmed,microbes-2019/habitats.full.txt -microorganisms pubmed,microbes-2019/microorganisms.full.txt -phenotype-relations pubmed,microbes-2019/phenotype-relations.full.txt -phenotype-relations pubmed,microbes-2019/phenotype-relations.txt -phenotypes pubmed,microbes-2019/phenotypes.full.txt -relations pubmed,microbes-2019/relations.full.txt -uses pubmed,microbes-2019/uses.full.txt -uses-relations pubmed,microbes-2019/uses-relations.full.txt -- GitLab From 3645e05fe6727c0a0f73521f3997c6e22e44283a Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 14:54:15 +0000 Subject: [PATCH 03/21] Update generate_stats.snakefile --- generate_stats.snakefile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/generate_stats.snakefile b/generate_stats.snakefile index 40c76264..24294c6c 100644 --- a/generate_stats.snakefile +++ b/generate_stats.snakefile @@ -36,12 +36,14 @@ rule stats_cirm_BIA: df2.to_csv(output.stats, index=False) ''' -cirm | nb entites | count_lines(corpora/cirm/mapped_taxids.txt) +cirm | nb entites | count_lines(corpora/cirm/mapped_bia_taxa.txt) cirm | nb yeast entities | count_lines(corpora/cirm/mapped_yeast_taxa.txt) -cirm | nb entites | count_lines(corpora/cirm/mapped_habitats.txt) +cirm | nb entites | count_lines(corpora/cirm/mapped_bia_habitats.txt) cirm | nb yeast habitats | count_lines(corpora/cirm/mapped_yeast_habitats.txt) +cirm | nb entites | count_lines(corpora/cirm/mapped_cfbp_habitats.txt) +cirm | nb yeast habitats | count_lines(corpora/cirm/mapped_cfbp_habitats.txt) ''' -SORTIES_CIRM= ["mapped_taxids.txt", "mapped_yeast_taxa.txt", "mapped_habitats.txt", "mapped_yeast_habitats.txt" ] +SORTIES_CIRM= ["mapped_bia_taxa.txt", "mapped_yeast_taxa.txt", "mapped_cfbp_taxa.txt", "mapped_bia_habitats.txt", "mapped_yeast_habitats.txt", "mapped_cfbp_habitats.txt" ] ''' ''' rule stats_cirm_Levure: @@ -126,7 +128,7 @@ dsmz | nb entites | count_lines(corpora/dsmz/mapped_taxids.txt) dsmz | nb habitats | count_lines(corpora/dsmz/mapped_habitats.txt) ''' ENTREES_DSMZ = ["dsmz-data/category=from_ncbi_taxonomy-key=taxid.tsv" ] -SORTIES_DSMZ = ["mapped_taxids.txt", "mapped_habitats.txt" ] +SORTIES_DSMZ = [ "mapped_habitats.txt" ] FILES_DSMZ = ENTREES_DSMZ + SORTIES_DSMZ ''' ''' -- GitLab From 9a40bd17ecfce91cf670940b114a30214c04973b Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 15:08:43 +0000 Subject: [PATCH 04/21] Update generate_stats.snakefile --- generate_stats.snakefile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/generate_stats.snakefile b/generate_stats.snakefile index 24294c6c..df88e56f 100644 --- a/generate_stats.snakefile +++ b/generate_stats.snakefile @@ -181,6 +181,17 @@ SORTIES_PUBMED = ["relations.full.txt", "phenotype-relations.full.txt", "uses-re FILES_PUBMED = ENTREES_PUBMED + SORTIES_PUBMED +''' +''' +rule get_list_of_batches: + input: + batches="corpora/pubmed/batches/" + output + list="corpora/pubmed/list_of_batches.txt" + shell:""" + ls {input.batches}/*/batch.xml >> {ouput.list} + """ + ''' ''' -- GitLab From 70bfe1435116df2ac3720045cc8f0e974e41b369 Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 15:09:37 +0000 Subject: [PATCH 05/21] Update generate_stats.snakefile --- generate_stats.snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate_stats.snakefile b/generate_stats.snakefile index df88e56f..aa17d806 100644 --- a/generate_stats.snakefile +++ b/generate_stats.snakefile @@ -186,7 +186,7 @@ FILES_PUBMED = ENTREES_PUBMED + SORTIES_PUBMED rule get_list_of_batches: input: batches="corpora/pubmed/batches/" - output + output: list="corpora/pubmed/list_of_batches.txt" shell:""" ls {input.batches}/*/batch.xml >> {ouput.list} -- GitLab From 278db9a0727524647edb5249b90cd698d7f9ee61 Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 15:10:27 +0000 Subject: [PATCH 06/21] Update generate_stats.snakefile --- generate_stats.snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate_stats.snakefile b/generate_stats.snakefile index aa17d806..83b7c314 100644 --- a/generate_stats.snakefile +++ b/generate_stats.snakefile @@ -189,7 +189,7 @@ rule get_list_of_batches: output: list="corpora/pubmed/list_of_batches.txt" shell:""" - ls {input.batches}/*/batch.xml >> {ouput.list} + ls {input.batches}/*/batch.xml >> {output.list} """ ''' -- GitLab From 214621797783d00da11a67553ce8300e6288e0f2 Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 15:20:48 +0000 Subject: [PATCH 07/21] Update corpora/florilege/dates.metadata --- corpora/florilege/dates.metadata | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 corpora/florilege/dates.metadata diff --git a/corpora/florilege/dates.metadata b/corpora/florilege/dates.metadata new file mode 100644 index 00000000..b4b423ae --- /dev/null +++ b/corpora/florilege/dates.metadata @@ -0,0 +1,7 @@ + +cirm +genbank +dsmz +pubmed +ontobitope +ncbi taxo -- GitLab From 4989fb62d9fa913916936d489165ca9e1cf130b7 Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 15:32:18 +0000 Subject: [PATCH 08/21] Update corpora/florilege/dates.metadata, corpora/florilege/labels.stats files --- corpora/florilege/dates.metadata | 14 ++++++++------ corpora/florilege/labels.stats | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/corpora/florilege/dates.metadata b/corpora/florilege/dates.metadata index b4b423ae..85a7d9b9 100644 --- a/corpora/florilege/dates.metadata +++ b/corpora/florilege/dates.metadata @@ -1,7 +1,9 @@ -cirm -genbank -dsmz -pubmed -ontobitope -ncbi taxo +source,uri,valeur +cirm,/db/maj/genbank/date,17-02-2021 +genbank,/db/maj/genbank/date,27-01-2021 +dsmz,/db/maj/dsmz/date,26-01-2018 +pubmed,/db/maj/pubmed/date,01-09-2021 +ontobitope,/db/maj/ontobitope/date, +eval,migale/evaluation/BB19/date,14-04-2021 +ncbi,migale/ncbi/taxo/date, diff --git a/corpora/florilege/labels.stats b/corpora/florilege/labels.stats index 9082013d..ef9a206d 100644 --- a/corpora/florilege/labels.stats +++ b/corpora/florilege/labels.stats @@ -40,3 +40,4 @@ eval_BB19-rel+ner_003, score sur la prédiction des Exhibits de BB19-rel+ner,Bio eval_BB19-kb+ner_001, mesure pour l'evaluation de BB19-kb+ner,BioNLP-OST-2019/BB19-kb+ner#Mesure eval_BB19-kb+ner_002, score moyen sur BB19-kb+ner,BioNLP-OST-2019/BB19-kb+ner#Standard_scoring ontobiotope_000, date de mise à jour de ontobiotope,/db/maj/pubmed/date +ncbi_000,date de mise à jour de la taxo ncbi,migale/ncbi/taxo/date -- GitLab From f982664bce1a77b8eb9971408560f391a59007fa Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 15:35:11 +0000 Subject: [PATCH 09/21] Update corpora/florilege/dates.metadata, corpora/florilege/labels.stats files --- corpora/florilege/dates.metadata | 4 ++-- corpora/florilege/labels.stats | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/corpora/florilege/dates.metadata b/corpora/florilege/dates.metadata index 85a7d9b9..3f1f97fc 100644 --- a/corpora/florilege/dates.metadata +++ b/corpora/florilege/dates.metadata @@ -4,6 +4,6 @@ cirm,/db/maj/genbank/date,17-02-2021 genbank,/db/maj/genbank/date,27-01-2021 dsmz,/db/maj/dsmz/date,26-01-2018 pubmed,/db/maj/pubmed/date,01-09-2021 -ontobitope,/db/maj/ontobitope/date, +ontobitope,/db/maj/ontobitope/date,09-04-2021 eval,migale/evaluation/BB19/date,14-04-2021 -ncbi,migale/ncbi/taxo/date, +ncbi,migale/ncbi/taxo/date,9-04-2021 diff --git a/corpora/florilege/labels.stats b/corpora/florilege/labels.stats index ef9a206d..387bffaa 100644 --- a/corpora/florilege/labels.stats +++ b/corpora/florilege/labels.stats @@ -39,5 +39,5 @@ eval_BB19-rel+ner_002, score sur la prédiction des Lives-In de BB19-rel+ner,Bio eval_BB19-rel+ner_003, score sur la prédiction des Exhibits de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Exhibits eval_BB19-kb+ner_001, mesure pour l'evaluation de BB19-kb+ner,BioNLP-OST-2019/BB19-kb+ner#Mesure eval_BB19-kb+ner_002, score moyen sur BB19-kb+ner,BioNLP-OST-2019/BB19-kb+ner#Standard_scoring -ontobiotope_000, date de mise à jour de ontobiotope,/db/maj/pubmed/date +ontobiotope_000, date de mise à jour de ontobiotope,/db/maj/ontobiotope/date ncbi_000,date de mise à jour de la taxo ncbi,migale/ncbi/taxo/date -- GitLab From 0d53e1716f36efc30d8e4844a1180d60b0b0e36c Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 15:40:22 +0000 Subject: [PATCH 10/21] Update generate_stats.snakefile, corpora/florilege/dates.meta files --- corpora/florilege/{dates.metadata => dates.meta} | 0 generate_stats.snakefile | 6 ++++-- 2 files changed, 4 insertions(+), 2 deletions(-) rename corpora/florilege/{dates.metadata => dates.meta} (100%) diff --git a/corpora/florilege/dates.metadata b/corpora/florilege/dates.meta similarity index 100% rename from corpora/florilege/dates.metadata rename to corpora/florilege/dates.meta diff --git a/generate_stats.snakefile b/generate_stats.snakefile index 83b7c314..5b90183b 100644 --- a/generate_stats.snakefile +++ b/generate_stats.snakefile @@ -357,7 +357,6 @@ rule merge_all: result.to_csv(output.result, index=False) - ''' merge ''' @@ -365,11 +364,14 @@ rule joint_stats: input: full_r="corpora/florilege/stats.full.csv", concepts="corpora/florilege/labels.stats" + dates="corpora/florilege/dates.meta" output: result="corpora/florilege/full_stats_with_labels.csv" run: import pandas df1=pandas.read_csv(input.concepts) df2=pandas.read_csv(input.full_r) - df = pandas.merge(df1, df2, on="uri", how="left") + df3=pandas.read_csv(input.dates) + frames = [df1, df_2] + df = pandas.merge(df1, pandas.concat(frames), on="uri", how="left") df.to_csv(output.result, index=False) -- GitLab From c14b509dec10f585268f6ac197f271f52b025773 Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 15:41:38 +0000 Subject: [PATCH 11/21] Update generate_stats.snakefile --- generate_stats.snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate_stats.snakefile b/generate_stats.snakefile index 5b90183b..822200f3 100644 --- a/generate_stats.snakefile +++ b/generate_stats.snakefile @@ -363,7 +363,7 @@ merge rule joint_stats: input: full_r="corpora/florilege/stats.full.csv", - concepts="corpora/florilege/labels.stats" + concepts="corpora/florilege/labels.stats", dates="corpora/florilege/dates.meta" output: result="corpora/florilege/full_stats_with_labels.csv" -- GitLab From 42920f323cbfcec40f73fb57800bece774bca4ef Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 15:45:22 +0000 Subject: [PATCH 12/21] Update generate_stats.snakefile --- generate_stats.snakefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/generate_stats.snakefile b/generate_stats.snakefile index 822200f3..8b19a7c5 100644 --- a/generate_stats.snakefile +++ b/generate_stats.snakefile @@ -369,9 +369,7 @@ rule joint_stats: result="corpora/florilege/full_stats_with_labels.csv" run: import pandas - df1=pandas.read_csv(input.concepts) - df2=pandas.read_csv(input.full_r) - df3=pandas.read_csv(input.dates) - frames = [df1, df_2] + frames = [pandas.read_csv(input.concepts), pandas.read_csv(input.dates)] + df1=pandas.read_csv(input.full_r) df = pandas.merge(df1, pandas.concat(frames), on="uri", how="left") df.to_csv(output.result, index=False) -- GitLab From 4133de90de3d2e2efb617449f84cb5f6efcede59 Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 15:49:38 +0000 Subject: [PATCH 13/21] Update generate_stats.snakefile --- generate_stats.snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generate_stats.snakefile b/generate_stats.snakefile index 8b19a7c5..a7c6783f 100644 --- a/generate_stats.snakefile +++ b/generate_stats.snakefile @@ -369,7 +369,7 @@ rule joint_stats: result="corpora/florilege/full_stats_with_labels.csv" run: import pandas - frames = [pandas.read_csv(input.concepts), pandas.read_csv(input.dates)] - df1=pandas.read_csv(input.full_r) + frames = [pandas.read_csv(input.full_r), pandas.read_csv(input.dates)] + df1=pandas.read_csv(input.concepts) df = pandas.merge(df1, pandas.concat(frames), on="uri", how="left") df.to_csv(output.result, index=False) -- GitLab From 19c021255816627cb3020b4041aa4c40880a369c Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 15:56:56 +0000 Subject: [PATCH 14/21] Update corpora/florilege/labels.stats, corpora/florilege/dates.meta files --- corpora/florilege/dates.meta | 2 +- corpora/florilege/labels.stats | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/corpora/florilege/dates.meta b/corpora/florilege/dates.meta index 3f1f97fc..fd191100 100644 --- a/corpora/florilege/dates.meta +++ b/corpora/florilege/dates.meta @@ -6,4 +6,4 @@ dsmz,/db/maj/dsmz/date,26-01-2018 pubmed,/db/maj/pubmed/date,01-09-2021 ontobitope,/db/maj/ontobitope/date,09-04-2021 eval,migale/evaluation/BB19/date,14-04-2021 -ncbi,migale/ncbi/taxo/date,9-04-2021 +ncbi,migale/maj/ncbi/taxo/date,9-04-2021 diff --git a/corpora/florilege/labels.stats b/corpora/florilege/labels.stats index 387bffaa..2e1ef135 100644 --- a/corpora/florilege/labels.stats +++ b/corpora/florilege/labels.stats @@ -1,5 +1,5 @@ id,libelle,uri -cirm_000, date de mise à jour des données cirm, /db/maj/genbank/date +cirm_000,date de mise à jour des données cirm, /db/maj/genbank/date cirm_001,nombre d'entrées de cirm-BIA,cirm/BIA_2021/florilege_export_final_17_02_21.xlsx cirm_002,nombre d'entrées de cirm-Levure,cirm/Levures_2021/Florilege_21012021.xlsx cirm_003,nombre de taxons de cirm-BIA,cirm/mapped_bia_taxa.txt @@ -9,7 +9,7 @@ cirm_006,nombre d'habitats de cirm-Levure,cirm/mapped_yeast_habitats.txt cirm_007,nombre d'entrées de cirm-cfpb,corpora/cirm/CFBP_2020/CFPB_22_sept_2020_Type.xlsx cirm_006,nombre de taxons de cirm-cfpb,cirm/mapped_cfbp_taxa.txt cirm_006,nombre d'habitats de cirm-cfpb,cirm/mapped_cfbp_habitats.txt -genbank_000, date de mise à jour des données genbank,/db/maj/genbank/date +genbank_000,date de mise à jour des données genbank,/db/maj/genbank/date genbank_001,nombre d'entrées de genbank,genbank/GenBank_extraction_20210127.tsv genbank_002,nombre de taxons de genbank,genbank/mapped_taxa.txt genbank_003,nombre d'entités du type #Habitat de genbank,genbank/mapped_habitats.txt @@ -39,5 +39,5 @@ eval_BB19-rel+ner_002, score sur la prédiction des Lives-In de BB19-rel+ner,Bio eval_BB19-rel+ner_003, score sur la prédiction des Exhibits de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Exhibits eval_BB19-kb+ner_001, mesure pour l'evaluation de BB19-kb+ner,BioNLP-OST-2019/BB19-kb+ner#Mesure eval_BB19-kb+ner_002, score moyen sur BB19-kb+ner,BioNLP-OST-2019/BB19-kb+ner#Standard_scoring -ontobiotope_000, date de mise à jour de ontobiotope,/db/maj/ontobiotope/date -ncbi_000,date de mise à jour de la taxo ncbi,migale/ncbi/taxo/date +ontobiotope_000,date de mise à jour de ontobiotope,/db/maj/ontobiotope/date +ncbi_000,date de mise à jour de la taxo ncbi,migale/maj/ncbi/taxo/date -- GitLab From 37c530d2e9636b874e7b73a3c9747eef0afb4836 Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 16:08:47 +0000 Subject: [PATCH 15/21] Update corpora/florilege/labels.stats, generate_stats.snakefile files --- corpora/florilege/labels.stats | 10 +++++----- generate_stats.snakefile | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/corpora/florilege/labels.stats b/corpora/florilege/labels.stats index 2e1ef135..daadec61 100644 --- a/corpora/florilege/labels.stats +++ b/corpora/florilege/labels.stats @@ -4,26 +4,26 @@ cirm_001,nombre d'entrées de cirm-BIA,cirm/BIA_2021/florilege_export_final_17_0 cirm_002,nombre d'entrées de cirm-Levure,cirm/Levures_2021/Florilege_21012021.xlsx cirm_003,nombre de taxons de cirm-BIA,cirm/mapped_bia_taxa.txt cirm_004,nombre de taxons de cirm-Levure,cirm/mapped_yeast_taxa.txt -cirm_005,nombre d'habitats de cirm-BIA,cirm/mapped_habitats.txt +cirm_005,nombre d'habitats de cirm-BIA,cirm/mapped_bia_habitats.txt cirm_006,nombre d'habitats de cirm-Levure,cirm/mapped_yeast_habitats.txt cirm_007,nombre d'entrées de cirm-cfpb,corpora/cirm/CFBP_2020/CFPB_22_sept_2020_Type.xlsx cirm_006,nombre de taxons de cirm-cfpb,cirm/mapped_cfbp_taxa.txt cirm_006,nombre d'habitats de cirm-cfpb,cirm/mapped_cfbp_habitats.txt genbank_000,date de mise à jour des données genbank,/db/maj/genbank/date genbank_001,nombre d'entrées de genbank,genbank/GenBank_extraction_20210127.tsv -genbank_002,nombre de taxons de genbank,genbank/mapped_taxa.txt +genbank_002,nombre de taxons de genbank,genbank/mapped_taxids.txt genbank_003,nombre d'entités du type #Habitat de genbank,genbank/mapped_habitats.txt dsmz_000, date de mise à jour des données de dsmz,/db/maj/dsmz/date dsmz_001,nombre d'entrées de dsmz,dsmz/dsmz-data/category=from_ncbi_taxonomy-key=taxid.tsv -dsmz_002,nombre de taxon venant de dsmz,dsmz/mapped_taxa.txt +dsmz_002,nombre de taxon venant de dsmz,dsmz/mapped_taxids.txt dsmz_003,nombre d'entités de type #Habitat de dsmz,dsmz/mapped_habitats.txt pubmed_000, date de mise à jour du corpus pubmed,/db/maj/pubmed/date pubmed_001,nombre de batches (x1000) pubmed,microbes-2019/list_of_batches.txt pubmed_002,nombre d'entités du type #Habitat de pubmed,microbes-2019/habitats.full.txt pubmed_003,nombre d'entités du type #Taxon pubmed,microbes-2019/microorganisms.full.txt pubmed_004,nombre d'entités du type #Phenotype de pubmed,microbes-2019/phenotypes.full.txt -pubmed_005,nombre de relations de type #Use pubmed,microbes-2019/uses.full.txt -pubmed_006,nombre de relations du type #Taxon-Habitat de pubmed,microbes-2019/habitat-relations.full.txt +pubmed_005,nombre d'entités du type #Use de pubmed,microbes-2019/uses.full.txt +pubmed_006,nombre de relations du type #Taxon-Habitat de pubmed,microbes-2019/relations.full.txt pubmed_007,nombre de relations du type #Phenotype-Taxon pubmed,microbes-2019/phenotype-relations.full.txt pubmed_008,nombre de relations du type #Use-Taxon pubmed,microbes-2019/uses-relations.full.txt eval_001, corpus utilisés, https://sites.google.com/view/bb-2019 diff --git a/generate_stats.snakefile b/generate_stats.snakefile index a7c6783f..4e1d2eb3 100644 --- a/generate_stats.snakefile +++ b/generate_stats.snakefile @@ -16,7 +16,7 @@ SOURCES=["cirm", "genbank", "dsmz", "pubmed", "BioNLP-OST-2019"] cirm | nb entrees | count_lines(corpora/cirm/BIA_2021/florilege_export_final_17_02_21.xlsx) cirm | nb yeast entrees | count_lines(corpora/cirm/Levures_2021/Florilege_21012021.xlsx) ''' -ENTREES_CIRM = ["BIA_2021/florilege_export_final_17_02_21.xlsx", "Levures_2021/Florilege_21012021.xlsx"] +ENTREES_CIRM = ["BIA_2021/florilege_export_final_17_02_21.xlsx", "Levures_2021/Florilege_21012021.xlsx", "CFBP_2020/CFPB_22_sept_2020_Type.xlsx"] rule stats_cirm_BIA: input: file="corpora/cirm/{file}" -- GitLab From 5eba87bc503fa9608994f1136473bb7211115da8 Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 16:18:57 +0000 Subject: [PATCH 16/21] Update corpora/florilege/labels.stats --- corpora/florilege/labels.stats | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/corpora/florilege/labels.stats b/corpora/florilege/labels.stats index daadec61..c50abc0a 100644 --- a/corpora/florilege/labels.stats +++ b/corpora/florilege/labels.stats @@ -1,5 +1,5 @@ id,libelle,uri -cirm_000,date de mise à jour des données cirm, /db/maj/genbank/date +cirm_000,date de mise à jour des données cirm,/db/maj/genbank/date cirm_001,nombre d'entrées de cirm-BIA,cirm/BIA_2021/florilege_export_final_17_02_21.xlsx cirm_002,nombre d'entrées de cirm-Levure,cirm/Levures_2021/Florilege_21012021.xlsx cirm_003,nombre de taxons de cirm-BIA,cirm/mapped_bia_taxa.txt @@ -26,18 +26,18 @@ pubmed_005,nombre d'entités du type #Use de pubmed,microbes-2019/uses.full.txt pubmed_006,nombre de relations du type #Taxon-Habitat de pubmed,microbes-2019/relations.full.txt pubmed_007,nombre de relations du type #Phenotype-Taxon pubmed,microbes-2019/phenotype-relations.full.txt pubmed_008,nombre de relations du type #Use-Taxon pubmed,microbes-2019/uses-relations.full.txt -eval_001, corpus utilisés, https://sites.google.com/view/bb-2019 -eval_002, date d'évaluation, migale/evaluation/BB19/date -eval_BB19-norm+ner_001, mesure pour l'evaluation de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Mesure -eval_BB19-norm+ner_002, score global sur la prédiction de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Standard_scoring -eval_BB19-norm+ner_002, score sur la prédiction des taxons de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Microorganism -eval_BB19-norm+ner_003, score sur la prédiction des phénotypes de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Phenotype -eval_BB19-norm+ner_004, score sur la prédiction des habitats de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Habitat -eval_BB19-rel+ner_001, mesure pour l'evaluation de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Mesure -eval_BB19-rel+ner_002, score global sur la prédiction de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Standard_scoring -eval_BB19-rel+ner_002, score sur la prédiction des Lives-In de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Lives_In -eval_BB19-rel+ner_003, score sur la prédiction des Exhibits de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Exhibits -eval_BB19-kb+ner_001, mesure pour l'evaluation de BB19-kb+ner,BioNLP-OST-2019/BB19-kb+ner#Mesure -eval_BB19-kb+ner_002, score moyen sur BB19-kb+ner,BioNLP-OST-2019/BB19-kb+ner#Standard_scoring +eval_001,corpus utilisés,https://sites.google.com/view/bb-2019 +eval_002,date d'évaluation,migale/evaluation/BB19/date +eval_BB19-norm+ner_001,mesure pour l'evaluation de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Mesure +eval_BB19-norm+ner_002,score global sur la prédiction de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Standard_scoring +eval_BB19-norm+ner_002,score sur la prédiction des taxons de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Microorganism +eval_BB19-norm+ner_003,score sur la prédiction des phénotypes de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Phenotype +eval_BB19-norm+ner_004,score sur la prédiction des habitats de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Habitat +eval_BB19-rel+ner_001,mesure pour l'evaluation de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Mesure +eval_BB19-rel+ner_002,score global sur la prédiction de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Standard_scoring +eval_BB19-rel+ner_002,score sur la prédiction des Lives-In de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Lives_In +eval_BB19-rel+ner_003,score sur la prédiction des Exhibits de BB19-rel+ner,BioNLP-OST-2019/BB19-rel+ner#Exhibits +eval_BB19-kb+ner_001,mesure pour l'evaluation de BB19-kb+ner,BioNLP-OST-2019/BB19-kb+ner#Mesure +eval_BB19-kb+ner_002,score moyen sur BB19-kb+ner,BioNLP-OST-2019/BB19-kb+ner#Standard_scoring ontobiotope_000,date de mise à jour de ontobiotope,/db/maj/ontobiotope/date ncbi_000,date de mise à jour de la taxo ncbi,migale/maj/ncbi/taxo/date -- GitLab From 79f33d0b1a4932966f19da4c70e59e2b71fdb78d Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 16:22:44 +0000 Subject: [PATCH 17/21] Update corpora/florilege/labels.stats --- corpora/florilege/labels.stats | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/corpora/florilege/labels.stats b/corpora/florilege/labels.stats index c50abc0a..1a2b5711 100644 --- a/corpora/florilege/labels.stats +++ b/corpora/florilege/labels.stats @@ -18,14 +18,14 @@ dsmz_001,nombre d'entrées de dsmz,dsmz/dsmz-data/category=from_ncbi_taxonomy-ke dsmz_002,nombre de taxon venant de dsmz,dsmz/mapped_taxids.txt dsmz_003,nombre d'entités de type #Habitat de dsmz,dsmz/mapped_habitats.txt pubmed_000, date de mise à jour du corpus pubmed,/db/maj/pubmed/date -pubmed_001,nombre de batches (x1000) pubmed,microbes-2019/list_of_batches.txt -pubmed_002,nombre d'entités du type #Habitat de pubmed,microbes-2019/habitats.full.txt -pubmed_003,nombre d'entités du type #Taxon pubmed,microbes-2019/microorganisms.full.txt -pubmed_004,nombre d'entités du type #Phenotype de pubmed,microbes-2019/phenotypes.full.txt -pubmed_005,nombre d'entités du type #Use de pubmed,microbes-2019/uses.full.txt -pubmed_006,nombre de relations du type #Taxon-Habitat de pubmed,microbes-2019/relations.full.txt -pubmed_007,nombre de relations du type #Phenotype-Taxon pubmed,microbes-2019/phenotype-relations.full.txt -pubmed_008,nombre de relations du type #Use-Taxon pubmed,microbes-2019/uses-relations.full.txt +pubmed_001,nombre de batches (x1000) pubmed,pubmed/list_of_batches.txt +pubmed_002,nombre d'entités du type #Habitat de pubmed,pubmed/habitats.full.txt +pubmed_003,nombre d'entités du type #Taxon pubmed,pubmed/microorganisms.full.txt +pubmed_004,nombre d'entités du type #Phenotype de pubmed,pubmed/phenotypes.full.txt +pubmed_005,nombre d'entités du type #Use de pubmed,pubmed/uses.full.txt +pubmed_006,nombre de relations du type #Taxon-Habitat de pubmed,pubmed/relations.full.txt +pubmed_007,nombre de relations du type #Phenotype-Taxon pubmed,pubmed/phenotype-relations.full.txt +pubmed_008,nombre de relations du type #Use-Taxon pubmed,pubmed/uses-relations.full.txt eval_001,corpus utilisés,https://sites.google.com/view/bb-2019 eval_002,date d'évaluation,migale/evaluation/BB19/date eval_BB19-norm+ner_001,mesure pour l'evaluation de BB19-norm+ner,BioNLP-OST-2019/BB19-norm+ner#Mesure -- GitLab From f72e4188efd9145c847e945b76e7d6e33743d35d Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 16:23:48 +0000 Subject: [PATCH 18/21] Update corpora/florilege/dates.meta --- corpora/florilege/dates.meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corpora/florilege/dates.meta b/corpora/florilege/dates.meta index fd191100..d0b166ff 100644 --- a/corpora/florilege/dates.meta +++ b/corpora/florilege/dates.meta @@ -4,6 +4,6 @@ cirm,/db/maj/genbank/date,17-02-2021 genbank,/db/maj/genbank/date,27-01-2021 dsmz,/db/maj/dsmz/date,26-01-2018 pubmed,/db/maj/pubmed/date,01-09-2021 -ontobitope,/db/maj/ontobitope/date,09-04-2021 +ontobitope,/db/maj/ontobiotope/date,09-04-2021 eval,migale/evaluation/BB19/date,14-04-2021 ncbi,migale/maj/ncbi/taxo/date,9-04-2021 -- GitLab From c72aab946cc18fea201af71f226686176483c6d2 Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 17:26:16 +0000 Subject: [PATCH 19/21] Update corpora/florilege/labels.stats, corpora/florilege/dates.meta files --- corpora/florilege/dates.meta | 1 + corpora/florilege/labels.stats | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/corpora/florilege/dates.meta b/corpora/florilege/dates.meta index d0b166ff..4f73dce5 100644 --- a/corpora/florilege/dates.meta +++ b/corpora/florilege/dates.meta @@ -6,4 +6,5 @@ dsmz,/db/maj/dsmz/date,26-01-2018 pubmed,/db/maj/pubmed/date,01-09-2021 ontobitope,/db/maj/ontobiotope/date,09-04-2021 eval,migale/evaluation/BB19/date,14-04-2021 +eval,corpus d'evaluation,https://sites.google.com/view/bb-2019,Bacteria Bitope 2019 ncbi,migale/maj/ncbi/taxo/date,9-04-2021 diff --git a/corpora/florilege/labels.stats b/corpora/florilege/labels.stats index 1a2b5711..47867232 100644 --- a/corpora/florilege/labels.stats +++ b/corpora/florilege/labels.stats @@ -7,8 +7,8 @@ cirm_004,nombre de taxons de cirm-Levure,cirm/mapped_yeast_taxa.txt cirm_005,nombre d'habitats de cirm-BIA,cirm/mapped_bia_habitats.txt cirm_006,nombre d'habitats de cirm-Levure,cirm/mapped_yeast_habitats.txt cirm_007,nombre d'entrées de cirm-cfpb,corpora/cirm/CFBP_2020/CFPB_22_sept_2020_Type.xlsx -cirm_006,nombre de taxons de cirm-cfpb,cirm/mapped_cfbp_taxa.txt -cirm_006,nombre d'habitats de cirm-cfpb,cirm/mapped_cfbp_habitats.txt +cirm_008,nombre de taxons de cirm-cfpb,cirm/mapped_cfbp_taxa.txt +cirm_009,nombre d'habitats de cirm-cfpb,cirm/mapped_cfbp_habitats.txt genbank_000,date de mise à jour des données genbank,/db/maj/genbank/date genbank_001,nombre d'entrées de genbank,genbank/GenBank_extraction_20210127.tsv genbank_002,nombre de taxons de genbank,genbank/mapped_taxids.txt -- GitLab From a8e84aa3db9fe198388e7451d78cf88fa9b07a5d Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 17:35:39 +0000 Subject: [PATCH 20/21] Update corpora/florilege/dates.meta --- corpora/florilege/dates.meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corpora/florilege/dates.meta b/corpora/florilege/dates.meta index 4f73dce5..42ca6be6 100644 --- a/corpora/florilege/dates.meta +++ b/corpora/florilege/dates.meta @@ -6,5 +6,5 @@ dsmz,/db/maj/dsmz/date,26-01-2018 pubmed,/db/maj/pubmed/date,01-09-2021 ontobitope,/db/maj/ontobiotope/date,09-04-2021 eval,migale/evaluation/BB19/date,14-04-2021 -eval,corpus d'evaluation,https://sites.google.com/view/bb-2019,Bacteria Bitope 2019 +eval,https://sites.google.com/view/bb-2019,Bacteria Bitope 2019 ncbi,migale/maj/ncbi/taxo/date,9-04-2021 -- GitLab From 86133f30260931b22b6cec211a3cc668df5fca69 Mon Sep 17 00:00:00 2001 From: Mouhamadou Ba <mandiayba@gmail.com> Date: Wed, 14 Apr 2021 17:39:42 +0000 Subject: [PATCH 21/21] Update corpora/florilege/labels.stats --- corpora/florilege/labels.stats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corpora/florilege/labels.stats b/corpora/florilege/labels.stats index 47867232..12383268 100644 --- a/corpora/florilege/labels.stats +++ b/corpora/florilege/labels.stats @@ -6,7 +6,7 @@ cirm_003,nombre de taxons de cirm-BIA,cirm/mapped_bia_taxa.txt cirm_004,nombre de taxons de cirm-Levure,cirm/mapped_yeast_taxa.txt cirm_005,nombre d'habitats de cirm-BIA,cirm/mapped_bia_habitats.txt cirm_006,nombre d'habitats de cirm-Levure,cirm/mapped_yeast_habitats.txt -cirm_007,nombre d'entrées de cirm-cfpb,corpora/cirm/CFBP_2020/CFPB_22_sept_2020_Type.xlsx +cirm_007,nombre d'entrées de cirm-cfpb,cirm/CFBP_2020/CFPB_22_sept_2020_Type.xlsx cirm_008,nombre de taxons de cirm-cfpb,cirm/mapped_cfbp_taxa.txt cirm_009,nombre d'habitats de cirm-cfpb,cirm/mapped_cfbp_habitats.txt genbank_000,date de mise à jour des données genbank,/db/maj/genbank/date -- GitLab