Add sociofillmore code, load dataset via private dataset repo

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.9
+WORKDIR /app
+ADD . /app
+RUN mkdir /nltk_data
+RUN mkdir /.allennlp
+RUN mkdir /.cache
+RUN mkdir /.local
+RUN chmod -R 777 /nltk_data
+RUN chmod -R 777 /.allennlp
+RUN chmod -R 777 /.cache
+RUN chmod -R 777 /.local
+RUN pip install -v -r requirements.combined.txt
+CMD ["sh", "docker_commands.sh"]

app.py

@@ -1,4 +0,0 @@
-import os
-from gradio_client import Client
-
-client = Client("responsibility-framing/sociofillmore", hf_token=os.getenv("HF_TOKEN"))

docker_commands.sh

@@ -0,0 +1,8 @@
+# cd spanfinder/
+# python -m sociolome.lome_webserver &
+# cd ..
+git clone https://gossminn:[email protected]/datasets/responsibility-framing/sociofillmore-datasets
+cp -r sociofillmore-datasets/data .
+cp -r sociofillmore-datasets/output .
+
+python -m sociofillmore.webapp.app 0.0.0.0

resources/active_frames_full.csv

@@ -0,0 +1,1229 @@
+frame,active,notes
+Abandonment,TRUE,
+Abounding_with,FALSE,
+Absorb_heat,FALSE,
+Abundance,FALSE,
+Abusing,TRUE,
+Access_scenario,FALSE,
+Accompaniment,FALSE,
+Accomplishment,TRUE,
+Accoutrements,FALSE,
+Accuracy,FALSE,
+Achieving_first,TRUE,
+Active_substance,TRUE,
+Activity,TRUE,
+Activity_abandoned_state,TRUE,
+Activity_done_state,TRUE,
+Activity_finish,TRUE,
+Activity_ongoing,TRUE,
+Activity_pause,TRUE,
+Activity_paused_state,TRUE,
+Activity_prepare,TRUE,
+Activity_ready_state,FALSE,
+Activity_resume,TRUE,
+Activity_start,TRUE,
+Activity_stop,TRUE,
+Actually_occurring_entity,FALSE,
+Addiction,FALSE,
+Adding_up,TRUE,
+Adducing,TRUE,
+Adjacency,FALSE,
+Adjusting,TRUE,
+Adopt_selection,TRUE,
+Aesthetics,FALSE,
+Affirm_or_deny,TRUE,
+Age,FALSE,
+Aggregate,FALSE,
+Aging,FALSE,
+Agree_or_refuse_to_act,TRUE,
+Agriculture,TRUE,
+Aiming,TRUE,
+Alignment_image_schema,FALSE,
+Alliance,FALSE,
+Alternatives,FALSE,
+Alternativity,FALSE,
+Amalgamation,FALSE,
+Amassing,TRUE,"according to def, Recipient arg ‘sometimes agentive’, but from EN examples seems unergative"
+Ambient_temperature,FALSE,
+Ammunition,FALSE,
+Amounting_to,FALSE,
+Animals,FALSE,
+Annoyance,FALSE,
+Appeal,TRUE,
+Appellations,FALSE,
+Apply_heat,TRUE,
+Appointing,TRUE,
+Architectural_part,FALSE,
+Arithmetic,FALSE,
+Arithmetic_commutative,FALSE,
+Arithmetic_non-commutative,FALSE,
+Armor,FALSE,
+Arraignment,FALSE,
+Arranging,TRUE,
+Arrest,TRUE,
+Arriving,FALSE,
+Arson,TRUE,
+Artifact,FALSE,
+Artifact_subpart,FALSE,
+Artificiality,FALSE,
+Artistic_style,FALSE,
+Assemble,TRUE,
+Assessing,TRUE,
+Assigned_location,FALSE,
+Assistance,TRUE,
+Asymmetric_reciprocality,TRUE,
+Atonement,TRUE,
+Attaching,TRUE,
+Attack,TRUE,
+Attempt,TRUE,
+Attempt_action_scenario,FALSE,
+Attempt_distant_interaction_scenario,FALSE,
+Attempt_means,TRUE,
+Attempt_obtain_food_scenario,FALSE,
+Attempt_obtain_mineral_scenario,FALSE,
+Attempt_suasion,TRUE,
+Attempting_and_resolving_scenario,FALSE,
+Attending,TRUE,
+Attention,TRUE,
+Attention_getting,FALSE,
+Attitude_description,FALSE,
+Attributed_information,FALSE,
+Attributes,FALSE,
+Authority,TRUE,
+Avoiding,TRUE,
+Awareness,TRUE,
+Awareness_change_scenario,FALSE,
+Awareness_situation,FALSE,
+Awareness_status,FALSE,
+Bail_decision,TRUE,
+Basis_for_attribute,FALSE,
+Be_in_agreement_on_action,TRUE,
+Be_in_agreement_on_assessment,TRUE,
+Be_on_alert,FALSE,
+Be_subset_of,FALSE,
+Be_translation_equivalent,FALSE,
+Bearing_arms,TRUE,
+Beat_opponent,TRUE,
+Becoming,FALSE,
+Becoming_a_member,TRUE,
+Becoming_attached,FALSE,
+Becoming_aware,TRUE,
+Becoming_detached,FALSE,
+Becoming_dry,FALSE,
+Becoming_separated,FALSE,
+Becoming_silent,TRUE,
+Becoming_visible,FALSE,
+Behind_the_scenes,TRUE,
+Being_active,TRUE,
+Being_at_risk,FALSE,
+Being_attached,FALSE,
+Being_awake,TRUE,
+Being_born,FALSE,
+Being_contained_within,FALSE,
+Being_detached,FALSE,
+Being_dry,FALSE,
+Being_employed,TRUE,
+Being_in_captivity,FALSE,
+Being_in_category,FALSE,
+Being_in_control,TRUE,
+Being_in_effect,FALSE,
+Being_in_operation,TRUE,
+Being_incarcerated,TRUE,e.g. ‘serve time’
+Being_included,FALSE,
+Being_located,TRUE,e.g. ‘sit on’
+Being_named,FALSE,
+Being_necessary,FALSE,
+Being_obligated,FALSE,
+Being_obligatory,FALSE,"? only verb ‘behoove’, not sure about properties"
+Being_operational,TRUE,"e.g. ‘works, is working’"
+Being_pregnant,FALSE,
+Being_questionable,FALSE,
+Being_relevant,FALSE,"? ‘pertain ‘seems ergative syntactically, but not sure"
+Being_rotted,FALSE,
+Being_up_to_it,FALSE,
+Being_wet,FALSE,
+Besieging,TRUE,
+Beyond_compare,FALSE,
+Billing,TRUE,
+Biological_area,FALSE,
+Biological_classification,FALSE,
+Biological_entity,FALSE,
+Biological_mechanisms,TRUE,
+Biological_urge,FALSE,
+Birth_scenario,FALSE,
+Board_vehicle,TRUE,
+Body_decoration,FALSE,
+Body_description_holistic,FALSE,
+Body_description_part,FALSE,
+Body_mark,FALSE,
+Body_movement,TRUE,
+Body_parts,FALSE,
+Bond_maturation,TRUE,
+Borrowing,TRUE,
+Boundary,FALSE,
+Bounded_entity,FALSE,
+Bounded_region,FALSE,
+Bragging,TRUE,
+Breaking_apart,FALSE,
+Breaking_off,FALSE,
+Breaking_out_captive,TRUE,
+Breathing,TRUE,
+Bringing,TRUE,
+Building,TRUE,
+Building_subparts,FALSE,
+Buildings,FALSE,
+Bungling,TRUE,
+Burying,TRUE,
+Business_closure,TRUE,'founder’ seems ambiguous (?foundered efforts? foundered business?) but ‘close doors’ is clearly unergative
+Businesses,FALSE,
+Cache,FALSE,
+Calendric_unit,FALSE,
+Candidness,FALSE,
+Capability,TRUE,
+Capacity,FALSE,verbs difficult to judge (e.g. ‘a stadium seats’ ??)
+Capital_stock,FALSE,
+Cardinal_numbers,FALSE,
+Carry_goods,TRUE,
+Catastrophe,TRUE,
+Catching_fire,FALSE,
+Categorization,TRUE,
+Causation,TRUE,
+Causation_scenario,TRUE,
+Cause_benefit_or_detriment,TRUE,
+Cause_bodily_experience,TRUE,
+Cause_change,TRUE,
+Cause_change_of_consistency,TRUE,
+Cause_change_of_phase,TRUE,
+Cause_change_of_position_on_a_scale,TRUE,
+Cause_change_of_strength,TRUE,
+Cause_emotion,TRUE,
+Cause_expansion,TRUE,
+Cause_fluidic_motion,TRUE,
+Cause_harm,TRUE,
+Cause_impact,TRUE,
+Cause_motion,TRUE,
+Cause_proliferation_in_number,TRUE,
+Cause_temperature_change,TRUE,
+Cause_to_amalgamate,TRUE,
+Cause_to_be_dry,TRUE,
+Cause_to_be_included,TRUE,
+Cause_to_be_sharp,TRUE,
+Cause_to_be_wet,TRUE,
+Cause_to_burn,TRUE,
+Cause_to_continue,TRUE,
+Cause_to_end,TRUE,
+Cause_to_experience,TRUE,
+Cause_to_fragment,TRUE,
+Cause_to_land,TRUE,
+Cause_to_make_noise,TRUE,
+Cause_to_make_progress,TRUE,
+Cause_to_move_in_place,TRUE,
+Cause_to_perceive,TRUE,
+Cause_to_resume,TRUE,
+Cause_to_rot,TRUE,
+Cause_to_start,TRUE,
+Cause_to_wake,TRUE,
+Ceasing_to_be,FALSE,
+Certainty,TRUE,
+Change_accessibility,TRUE,
+Change_direction,TRUE,"1st FE is called Theme, but verbs (‘turn’) pass test"
+Change_event_duration,TRUE,
+Change_event_time,TRUE,
+Change_of_consistency,FALSE,
+Change_of_leadership,TRUE,Selector is not 1st FE but seems to be the subj of all of the verbs in the frame
+Change_of_phase,FALSE,'the lake froze’ → ‘the frozen lake’ => unaccusative
+Change_of_phase_scenario,FALSE,
+Change_of_quantity_of_possession,TRUE,
+Change_of_temperature,FALSE,
+Change_operational_state,TRUE,
+Change_position_on_a_scale,FALSE,
+Change_post-state,FALSE,
+Change_posture,TRUE,
+Change_resistance,TRUE,
+Change_tool,TRUE,
+Chaos,FALSE,
+Chatting,TRUE,
+Chemical-sense_description,TRUE,
+Chemical_potency,FALSE,
+Choosing,TRUE,
+Circumscribed_existence,FALSE,
+Citing,TRUE,
+Claim_ownership,TRUE,
+Clemency,FALSE,no verbs but describes active event
+Closure,TRUE,
+Clothing,FALSE,
+Clothing_parts,FALSE,
+Co-association,FALSE,
+Cogitation,TRUE,
+Cognitive_connection,TRUE,
+Coincidence,FALSE,
+Collaboration,TRUE,
+Collocation_image_schema,FALSE,
+Colonization,TRUE,
+Color,FALSE,
+Color_qualities,FALSE,
+Come_down_with,TRUE,
+Come_into_effect,FALSE,"not sure, ‘terms and conditions apply’ → ‘the applied terms and conditions’?"
+Come_together,TRUE,
+Coming_to_be,FALSE,
+Coming_to_believe,TRUE,
+Coming_up_with,TRUE,
+Commemorative,FALSE,
+Commerce_buy,TRUE,
+Commerce_collect,TRUE,
+Commerce_goods-transfer,TRUE,
+Commerce_money-transfer,TRUE,
+Commerce_pay,TRUE,
+Commerce_scenario,TRUE,
+Commerce_sell,TRUE,
+Commercial_transaction,TRUE,
+Commitment,TRUE,
+Committing_crime,TRUE,
+Commonality,FALSE,
+Communicate_categorization,TRUE,
+Communication,TRUE,
+Communication_manner,TRUE,
+Communication_means,TRUE,
+Communication_noise,TRUE,
+Communication_response,TRUE,
+Commutation,TRUE,
+Commutative_process,TRUE,"hard to judge, no examples"
+Commutative_statement,FALSE,
+Compatibility,FALSE,"Difficult, but it’s 'rhyming words’ not ‘rhymed words’; ‘matched’/’matching’ both possible but ‘matched’ implies an agent (probably a different frame)"
+Competition,TRUE,
+Complaining,TRUE,
+Completeness,FALSE,
+Compliance,TRUE,e.g. ‘obey’
+Concessive,FALSE,
+Condition_symptom_relation,TRUE,
+Conditional_occurrence,FALSE,
+Conditional_scenario,FALSE,
+Conduct,TRUE,
+Confronting_problem,TRUE,
+Connecting_architecture,FALSE,
+Connectors,FALSE,
+Conquering,TRUE,
+Contact_image_schema,FALSE,
+Contacting,TRUE,
+Container_focused_placing,TRUE,
+Container_focused_removing,TRUE,
+Containers,FALSE,
+Containing,TRUE,
+Containment_scenario,FALSE,
+Contingency,TRUE,
+Continued_state_of_affairs,FALSE,
+Contrary_circumstances,FALSE,
+Contrition,TRUE,
+Control,TRUE,
+Controller_object,FALSE,
+Convey_importance,TRUE,
+Convoy,FALSE,
+Cooking_creation,TRUE,
+Corporal_punishment,TRUE,
+Correctness,FALSE,
+Corroding,FALSE,
+Corroding_caused,TRUE,
+Cotheme,TRUE,"1st FE is called Theme, but verbs (e.g. ‘follow’) seem agentive"
+Counterattack,TRUE,
+Court_examination,TRUE,
+Craft,FALSE,
+Create_physical_artwork,TRUE,
+Create_representation,TRUE,
+Creating,TRUE,
+Crime_scenario,FALSE,
+Criminal_investigation,TRUE,
+Criminal_process,FALSE,
+Cure,TRUE,
+Custom,FALSE,
+Cutting,TRUE,
+Cycle_of_existence_scenario,FALSE,
+Cycle_of_life_and_death,FALSE,
+Damaging,TRUE,
+Daring,TRUE,
+Dead_or_alive,FALSE,
+Death,FALSE,
+Deception_end,TRUE,
+Deception_scenario,FALSE,
+Deception_success,TRUE,
+Deciding,TRUE,
+Defending,TRUE,
+Degree,FALSE,
+Degree_of_processing,FALSE,
+Delimitation_of_diversity,FALSE,
+Delimited_state_scenario,FALSE,
+Delivery,TRUE,
+Deny_or_grant_permission,TRUE,
+Departing,TRUE,Directional movement verbs
+Deserving,TRUE,"verbs don’t seem to be able to be used intransitively, but seem ‘active-like’ (e.g. ‘justify’)"
+Desirability,TRUE,? e.g. ‘the movie rocks/sucks’ (metaphorical but derives from active verb?)
+Desirable_event,TRUE,
+Desiring,TRUE,
+Destiny,FALSE,
+Destroying,TRUE,
+Detaching,TRUE,
+Detaining,TRUE,
+Detonate_explosive,TRUE,
+Differentiation,TRUE,
+Difficulty,FALSE,
+Dimension,FALSE,"only one verb (‘measure’), cannot be intransitive"
+Direction,FALSE,
+Directional_locative_relation,FALSE,
+Disaster_scenario,FALSE,
+Discussion,TRUE,
+Disembarking,TRUE,Directional movement verbs
+Disgraceful_situation,FALSE,
+Dispersal,TRUE,
+Distance_scenario,FALSE,
+Distant_operated_IED,FALSE,
+Distinctiveness,TRUE,'characterize’
+Distributed_abundanced,FALSE,
+Distributed_position,TRUE,?? not sure any of the verbs can ever be intransitive
+Diversity,FALSE,
+Documents,FALSE,
+Dodging,TRUE,
+Domain,FALSE,
+Dominate_competitor,TRUE,
+Dominate_situation,TRUE,
+Domination,TRUE,
+Dough_rising,FALSE,
+Downing,TRUE,
+Dressing,TRUE,
+Drop_in_on,TRUE,
+Dunking,TRUE,
+Duplication,TRUE,
+Duration_description,FALSE,
+Duration_relation,FALSE,'it lasted’ → *’the lasted thing’ (but ??’the persisted thing’)
+Duration_scenario,FALSE,
+Dying,FALSE,
+Dynamic_situation_scenario,FALSE,
+Dynamism,FALSE,
+Earnings_and_losses,TRUE,
+Eclipse,TRUE,
+Economy,FALSE,
+Education_teaching,TRUE,
+Electricity,FALSE,
+Elusive_goal,FALSE,
+Emanating,FALSE,
+Emergency,FALSE,
+Emergency_fire,FALSE,
+Emitting,TRUE,
+Emotion_directed,TRUE,
+Emotion_heat,TRUE,
+Emotions,FALSE,
+Emotions_by_possibility,FALSE,
+Emotions_by_stimulus,FALSE,
+Emotions_of_mental_activity,TRUE,
+Emotions_success_or_failure,FALSE,Contains no verbs but theoretically possible that it would?
+Emphasizing,TRUE,
+Employee_scenario,FALSE,
+Employer_scenario,FALSE,
+Employing,TRUE,
+Employment_continue,FALSE,
+Employment_end,FALSE,
+Employment_scenario,FALSE,
+Employment_start,FALSE,
+Emptying,TRUE,
+Encoding,TRUE,
+Encounter,TRUE,
+Endangering,TRUE,
+Endeavor_failure,FALSE,
+Enforcing,TRUE,
+Enter_awareness,TRUE,
+Entering_of_plea,TRUE,?
+Entity,FALSE,
+Entourage,FALSE,
+Erasing,TRUE,
+Escaping,TRUE,
+Estimated_value,FALSE,
+Estimating,TRUE,
+Evading,TRUE,
+Evaluative_comparison,TRUE,
+Event,FALSE,
+Event_endstate,FALSE,
+Event_initial_state,FALSE,
+Event_instance,FALSE,
+Eventive_affecting,FALSE,
+Eventive_cognizer_affecting,TRUE,
+Evidence,TRUE,
+Evoking,TRUE,
+Examination,TRUE,
+Exchange,TRUE,
+Exchange_currency,TRUE,
+Exclude_member,TRUE,
+Excreting,TRUE,
+Execute_plan,TRUE,
+Execution,TRUE,
+Exemplar,FALSE,
+Exemplariness,FALSE,
+Exercising,TRUE,
+Existence,FALSE,
+Expansion,FALSE,
+Expectation,TRUE,
+Expected_location_of_person,FALSE,
+Expend_resource,TRUE,
+Expensiveness,TRUE,
+Experience_bodily_harm,FALSE,syntactically ambiguous? ‘I broke my leg’ / ‘mi sono rotto la gamba’ / ‘je me suis cassé la jambe’ → not sure how to classify this construction in romance languages?
+Experiencer_focused_emotion,TRUE,
+Experimentation,TRUE,
+Expertise,TRUE,
+Explaining_the_facts,TRUE,
+Explosion,FALSE,
+Exporting,TRUE,
+Expressing_publicly,TRUE,
+Extradition,TRUE,
+Extreme_point,FALSE,
+Extreme_value,FALSE,
+Facial_expression,FALSE,
+Fairness_evaluation,FALSE,
+Fall_asleep,FALSE,
+Fall_for,TRUE,"'buy’, ‘swallow’ seem clear transitive, ‘fall for’ could maybe be unaccusative??"
+Fame,TRUE,
+Familiarity,TRUE,
+Fastener,FALSE,
+Fear,TRUE,
+Feeling,TRUE,
+Feigning,TRUE,
+Fields,FALSE,
+Fighting_activity,FALSE,
+Filling,TRUE,
+Fining,TRUE,
+Finish_competition,TRUE,
+Finish_game,TRUE,
+Fire_break,FALSE,
+Fire_burning,TRUE,
+Fire_emergency_scenario,FALSE,
+Fire_end_scenario,FALSE,
+Fire_going_out,FALSE,
+Fire_stopping_scenario,FALSE,
+Firefighting,TRUE,
+Firing,TRUE,
+Firing_point,FALSE,
+First_experience,FALSE,
+First_rank,FALSE,
+Fleeing,TRUE,
+Fluidic_motion,TRUE,
+Food,FALSE,
+Food_gathering,TRUE,
+Foreign_or_domestic_country,FALSE,
+Forging,TRUE,
+Forgiveness,TRUE,
+Forgoing,TRUE,
+Forming_relationships,TRUE,
+Fragmentation_scenario,FALSE,
+Freeing_from_confinement,TRUE,
+Frequency,FALSE,
+Friction,TRUE,"difficult, but “Theme exterts pressure and experiences resistance” could be interpreted as somehow (quasi-)agentive"
+Friendly_or_hostile,FALSE,
+Front_for,TRUE,(front.v ?)
+Frugality,TRUE,
+Fugitive,FALSE,
+Fullness,FALSE,
+Function,TRUE,"difficult: inanimate theme, but “exists to perform Activity” so perhaps (metaphorically) can be seen as active?"
+Funding,TRUE,
+Gathering_up,TRUE,
+Gesture,TRUE,
+Get_a_job,TRUE,
+Getting,TRUE,"somewhat difficult: most verbs (obtain, acquire, procure) imply active action, but the definition of the frame and verbs like “get” can be seen as passive. 
+
+Interesting: in Dutch there is “verkrijgen” (obtain) vs. “krijgen” (get) were this distinction is encoded morphologically"
+Getting_scenario,FALSE,
+Getting_triggered,FALSE,
+Getting_underway,TRUE,
+Getting_up,TRUE,
+Getting_vehicle_underway,TRUE,
+Give_impression,TRUE,not sure but these all seem (semantically) causative somehow? e.g. “smell good” ~= “cause pleasant taste perception”?
+Giving,TRUE,
+Giving_birth,TRUE,
+Giving_in,TRUE,
+Giving_scenario,FALSE,
+Gizmo,FALSE,
+Go_into_shape,FALSE,"per description “Theme goed into a shape without being made to do so by an agent”, but does that make it agentive?"
+Goal,FALSE,
+Going_back_on_a_commitment,TRUE,
+Government_institution,FALSE,
+Gradable_artistic_quality,FALSE,
+Gradable_attributes,FALSE,
+Gradable_proximity,FALSE,
+Graph_shape,FALSE,
+Grasp,TRUE,
+Grinding,TRUE,
+Grooming,TRUE,
+Ground_up,FALSE,
+Growing_food,TRUE,
+Guest_and_host,FALSE,
+Guilt_or_innocence,FALSE,
+Gusto,FALSE,
+Hair_configuration,FALSE,
+Halt,TRUE,"motion verbs (‘has stopped’ etc)
+
+'stop’ seems to imply agent? ‘the cyclist stopped’, ‘the train stopped’ (metaphorical?), but ??the falling rock stopped (?)"
+Have_as_requirement,TRUE,"metaphorical use of demand, require, take?"
+Have_as_translation_equivalent,FALSE,
+Have_associated,FALSE,metaphorical use of have?
+Have_visitor_over,FALSE,
+Having_commercial_agreement,FALSE,
+Having_or_lacking_access,FALSE,"verb ‘access’ is agentive but seems to be misclassified in this frame (it means to enter, not to have access)"
+Health_response,FALSE,
+Hearsay,TRUE,
+Heat_potential,FALSE,
+Hedging,FALSE,
+Heralding,TRUE,
+Hiding_objects,TRUE,
+Hindering,TRUE,
+Hiring,TRUE,
+Historic_event,FALSE,
+History,FALSE,
+History_scenario,FALSE,
+Hit_or_miss,TRUE,
+Hit_target,TRUE,
+Holding_off_on,TRUE,
+Hospitality,FALSE,
+Hostile_encounter,TRUE,
+Hunting,TRUE,
+Hunting_scenario,FALSE,
+Hunting_success_or_failure,TRUE,
+Identicality,FALSE,
+Identity,FALSE,
+Identity_scenario,FALSE,
+Idiosyncrasy,FALSE,
+Image_schema,FALSE,
+Imitating,TRUE,
+Immobilization,TRUE,
+Impact,TRUE,
+Import_export_scenario,TRUE,
+Importance,TRUE,
+Importing,TRUE,
+Imposing_obligation,TRUE,
+Impression,FALSE,
+Imprisonment,TRUE,
+Improvement_or_decline,FALSE,
+Improvised_explosive_device,FALSE,
+Inclination,FALSE,
+Inclusion,TRUE,
+Inclusion_scenario,FALSE,
+Increment,FALSE,
+Indicating,TRUE,
+Indigenous_origin,FALSE,
+Individual_history,FALSE,
+Ineffability,FALSE,
+Infecting,TRUE,
+Influencing_potential,FALSE,
+Information,FALSE,
+Information_display,FALSE,
+Infrastructure,FALSE,
+Ingest_substance,TRUE,
+Ingestion,TRUE,
+Ingredients,FALSE,
+Inherent_purpose,FALSE,
+Inhibit_motion_scenario,FALSE,
+Inhibit_movement,TRUE,
+Inspecting,TRUE,
+Installing,TRUE,
+Instance,FALSE,
+Institutionalization,TRUE,
+Institutions,FALSE,
+Intentional_deception,TRUE,
+Intentional_traversing,TRUE,
+Intentionally_act,TRUE,
+Intentionally_affect,TRUE,
+Intentionally_create,TRUE,
+Intercepting,TRUE,
+Interior_profile_relation,FALSE,
+Interrupt_process,TRUE,
+Intoxicants,FALSE,
+Intoxication,FALSE,
+Invading,TRUE,
+Invasion_scenario,FALSE,
+Irregular_combatants,FALSE,
+Isolated_places,FALSE,
+Judgment,TRUE,
+Judgment_communication,TRUE,
+Judgment_direct_address,TRUE,
+Judgment_of_intensity,FALSE,
+Judicial_body,FALSE,
+Jury_deliberation,TRUE,
+Just_found_out,FALSE,
+Justifying,TRUE,
+Key,FALSE,
+Kidnapping,TRUE,
+Killing,TRUE,
+Kinship,FALSE,
+Knot_creation,TRUE,
+Knot_creation_scenario,FALSE,
+Labeling,TRUE,
+Labor_product,FALSE,
+Launch_process,TRUE,
+Law,FALSE,
+Law_enforcement_agency,FALSE,
+Leadership,TRUE,
+Leaving_traces,FALSE,
+Left_to_do,FALSE,
+Legal_rulings,TRUE,
+Legality,FALSE,
+Lending,TRUE,
+Level_of_force_exertion,FALSE,
+Level_of_force_resistance,FALSE,
+Level_of_light,FALSE,
+Light_movement,TRUE,metaphorical
+Likelihood,FALSE,
+Limitation,FALSE,
+Limiting,TRUE,
+Linguistic_meaning,TRUE,? metaphorical active?
+Lively_place,TRUE,? ‘the park buzzes’ can be seen as metaphorically active?
+Living_conditions,FALSE,
+Locale,FALSE,
+Locale_by_characteristic_entity,FALSE,
+Locale_by_collocation,FALSE,
+Locale_by_event,FALSE,
+Locale_by_ownership,FALSE,
+Locale_by_use,FALSE,
+Locale_closure,FALSE,
+Locating,TRUE,
+Location_in_time,FALSE,
+Location_of_light,FALSE,seems anticausative of Light_movement?
+Location_on_path,FALSE,
+Locative_relation,TRUE,
+Locative_scenario,FALSE,
+Lodging_scenario,FALSE,
+Lose_possession,FALSE,
+Lose_possession_scenario,FALSE,
+Losing,TRUE,'losing sth’ could be both passive or active; in Dutch “heb verloren/ben verloren” both possible (sensitive to the contrast?)
+Losing_it,TRUE,
+Losing_someone,TRUE,
+Losing_track_of,FALSE,
+Losing_track_of_perceiver,TRUE,
+Losing_track_of_theme,TRUE,
+Luck,FALSE,
+Make_acquaintance,TRUE,
+Make_agreement_on_action,TRUE,
+Make_cognitive_connection,TRUE,
+Make_compromise,TRUE,
+Make_noise,TRUE,
+Making_arrangements,TRUE,
+Making_faces,TRUE,
+Manipulate_into_doing,TRUE,
+Manipulate_into_shape,TRUE,
+Manipulation,TRUE,
+Manner,FALSE,
+Manner_of_life,TRUE,
+Manufacturing,TRUE,
+Margin_of_resolution,FALSE,
+Mass_motion,FALSE,?
+Mathematical_relationship,FALSE,
+Means,FALSE,
+Measurable_attributes,FALSE,
+Measure_area,FALSE,
+Measure_by_action,FALSE,
+Measure_duration,FALSE,
+Measure_mass,FALSE,
+Measure_of_distance_and_length,FALSE,
+Measure_scenario,FALSE,
+Measure_volume,FALSE,
+Measures,FALSE,
+Medical_conditions,FALSE,
+Medical_instruments,FALSE,
+Medical_interaction_scenario,FALSE,
+Medical_intervention,TRUE,
+Medical_professionals,FALSE,
+Medical_specialties,FALSE,
+Medium,FALSE,
+Meet_specifications,TRUE,
+Meet_with,TRUE,
+Meet_with_response,TRUE,? ‘meet with critizism’ ~= elicit critizism?
+Member_of_military,FALSE,
+Membership,FALSE,? belong
+Memorization,TRUE,
+Memory,TRUE,
+Mental_activity,FALSE,
+Mental_property,FALSE,
+Mental_stimulus_exp_focus,FALSE,
+Mental_stimulus_stimulus_focus,FALSE,
+Mention,TRUE,
+Military,FALSE,
+Military_operation,TRUE,
+Mining,TRUE,
+Misdeed,TRUE,
+Money,FALSE,
+Morality_evaluation,FALSE,
+Motion,FALSE,"? this frame is meant to have the non-active versions of the motion verbs (as opposed to Self_motion, Operate_vehicle etc), but in practice it seems many examples are actually active"
+Motion_directional,FALSE,?
+Motion_noise,FALSE,?
+Motion_scenario,FALSE,
+Moving_in_place,FALSE,?
+Name_conferral,TRUE,
+Namesake,FALSE,
+Natural_features,FALSE,
+Needing,TRUE,
+Negation,FALSE,
+Negative_conditional,FALSE,
+Network,FALSE,
+Noise_makers,FALSE,
+Non-commutative_process,TRUE,active when Calculator (non-core) is present; there are no examples so not clear how often this is the case
+Non-commutative_statement,FALSE,
+Non-gradable_proximity,FALSE,
+Noncombatant,FALSE,
+Notability,FALSE,
+Notification_of_charges,TRUE,
+Nuclear_process,FALSE,
+Objective_influence,TRUE,
+Obligation_scenario,FALSE,
+Obscurity,FALSE,
+Obviousness,FALSE,
+Occupy_rank,FALSE,'he ranks second’ ~= ‘he is ranked second’
+Offenses,FALSE,
+Offering,TRUE,
+Offshoot,FALSE,
+Omen,TRUE,Predictive_phenomenon (quasi-actively) provides cues for something
+Ontogeny,FALSE,
+Openness,FALSE,
+Operate_vehicle,TRUE,
+Operate_vehicle_scenario,FALSE,
+Operating_a_system,TRUE,
+Operational_testing,TRUE,
+Opinion,TRUE,
+Opportunity,FALSE,
+Optical_image,FALSE,
+Ordinal_numbers,FALSE,
+Organization,FALSE,
+Origin,FALSE,
+Others_situation_as_stimulus,TRUE,
+Out_of_existence,FALSE,
+Pardon,TRUE,
+Part_edge,FALSE,
+Part_inner_outer,FALSE,
+Part_ordered_segments,FALSE,
+Part_orientational,FALSE,
+Part_piece,FALSE,
+Part_whole,FALSE,
+Partiality,TRUE,
+Participation,TRUE,
+Partitive,FALSE,
+Passing,TRUE,
+Passing_off,TRUE,
+Path_shape,FALSE,? are syntactically unergative
+Path_traveled,FALSE,
+Patrolling,TRUE,
+Pattern,FALSE,
+People,FALSE,
+People_along_political_spectrum,FALSE,
+People_by_age,FALSE,
+People_by_jurisdiction,FALSE,
+People_by_military_specialty,FALSE,
+People_by_morality,FALSE,
+People_by_origin,FALSE,
+People_by_religion,FALSE,
+People_by_residence,FALSE,
+People_by_vocation,FALSE,
+Perception,FALSE,
+Perception_active,TRUE,
+Perception_body,TRUE,? 'my head hurts’ → ‘my head causes pain sensation’ (?)
+Perception_experience,TRUE,"? these are ‘passive experience’ verbs, but syntactically active/unergative; you can argue this kind of perception still implies active processing on the part of the experiencer"
+Performers,FALSE,
+Performers_and_roles,TRUE,"'feature.v’ is an odd one because agent/patient are reversed (NB only one annotated example, Fes seem not correct)"
+Performing_arts,FALSE,
+Personal_relationship,TRUE,"Mostly nouns/adj, but verbs 'sleep with’, ‘befriend’ are active"
+Personal_success,TRUE,
+Physical_artworks,FALSE,
+Physical_entity,FALSE,
+Piracy,TRUE,
+Placing,TRUE,
+Placing_scenario,FALSE,
+Planned_trajectory,FALSE,
+Planting,TRUE,
+Plants,FALSE,
+Point_of_dispute,FALSE,
+Political_actions,TRUE,
+Political_locales,FALSE,
+Popularity,FALSE,
+Posing_as,TRUE,
+Position_on_a_scale,FALSE,
+Possession,TRUE,? owning as action?
+Possibility,FALSE,
+Post_getting,FALSE,
+Post_giving,FALSE,
+Post_lose_possession,FALSE,
+Post_receiving,FALSE,
+Post_transfer,FALSE,
+Posture,TRUE,
+Practice,TRUE,
+Praiseworthiness,FALSE,
+Prank,FALSE,
+Pre_getting,FALSE,
+Pre_giving,FALSE,
+Pre_lose_possession,FALSE,
+Pre_receiving,FALSE,
+Pre_transfer,FALSE,
+Precariousness,TRUE,only verb teeter.v
+Precipitation,FALSE,
+Predicament,FALSE,
+Predicting,TRUE,
+Preference,TRUE,
+Preferred_alternative_scenario,FALSE,
+Preliminaries,FALSE,
+Presence,FALSE,
+Presentation_of_mitigation,FALSE,
+Preserving,TRUE,
+Prevarication,TRUE,
+Prevent_or_allow_possession,TRUE,
+Preventing_or_letting,TRUE,
+Price_per_unit,FALSE,
+Prison,FALSE,
+Probability,FALSE,
+Process,FALSE,
+Process_completed_state,FALSE,
+Process_continue,FALSE,
+Process_end,FALSE,
+Process_initial_state,FALSE,
+Process_pause,FALSE,
+Process_resume,FALSE,
+Process_start,FALSE,
+Process_stop,FALSE,
+Process_stopped_state,FALSE,
+Process_uncompleted_state,FALSE,
+Processing_materials,TRUE,
+Procreative_sex,TRUE,
+Product_delivery,FALSE,
+Product_development,TRUE,
+Product_development_scenario,FALSE,
+Product_line,FALSE,
+Progression,FALSE,
+Prohibiting_or_licensing,TRUE,
+Project,FALSE,
+Proliferating_in_number,FALSE,
+Prominence,FALSE,
+Proper_reference,FALSE,
+Proportion,FALSE,
+Proportional_quantity,FALSE,
+Protecting,TRUE,
+Protest,TRUE,
+Provide_lodging,TRUE,
+Proximity_image_schema,FALSE,
+Public_services,FALSE,
+Publishing,TRUE,
+Punctual_perception,FALSE,
+Purpose,FALSE,
+Putting_out_fire,TRUE,
+Quantified_mass,FALSE,
+Quantity,FALSE,
+Quarreling,TRUE,
+Questioning,TRUE,
+Quitting,TRUE,
+Quitting_a_place,TRUE,
+Race_descriptor,FALSE,
+Range,FALSE,
+Rank,FALSE,
+Ranked_expectation,FALSE,
+Rape,TRUE,
+Rashness,FALSE,
+Rate_description,FALSE,
+Rate_quantification,FALSE,
+Ratification,TRUE,
+Reading_activity,TRUE,
+Reading_aloud,TRUE,
+Reading_perception,TRUE,
+Reason,FALSE,
+Reasoning,TRUE,
+Reassuring,TRUE,
+Rebellion,TRUE,
+Receive_visitor_scenario,FALSE,
+Receiving,TRUE,
+Receiving_scenario,FALSE,
+Reciprocality,FALSE,
+Recording,TRUE,
+Records,FALSE,
+Recovery,FALSE,
+Redirecting,TRUE,
+Reference_text,FALSE,
+Referring_by_name,TRUE,
+Reforming_a_system,TRUE,
+Regard,TRUE,
+Region_with_portal,FALSE,
+Reject_leadership,TRUE,does not exist in NLTK
+Rejuvenation,TRUE,
+Relating_concepts,TRUE,?
+Relation,FALSE,
+Relation_between_individuals,FALSE,
+Relational_location,FALSE,
+Relational_natural_features,FALSE,
+Relational_political_locales,FALSE,
+Relational_quantity,FALSE,
+Relative_time,FALSE,?
+Releasing,TRUE,
+Releasing_from_custody,FALSE,
+Reliance,TRUE,?
+Reliance_on_expectation,TRUE,
+Religious_belief,TRUE,
+Remainder,FALSE,
+Remembering_experience,TRUE,
+Remembering_information,TRUE,
+Remembering_to_do,TRUE,
+Removing,TRUE,
+Removing_scenario,FALSE,
+Render_nonfunctional,TRUE,
+Renting,TRUE,
+Renting_out,TRUE,
+Renunciation,TRUE,
+Reparation,TRUE,
+Repayment,TRUE,
+Repel,TRUE,
+Replacing,TRUE,
+Reporting,TRUE,
+Representative,FALSE,
+Representing,TRUE,?
+Request,TRUE,
+Request_entity,TRUE,
+Required_event,TRUE,?
+Requirement_scenario,FALSE,
+Rescuing,TRUE,
+Research,TRUE,
+Reserving,TRUE,
+Reshaping,TRUE,
+Residence,TRUE,
+Resolve_problem,TRUE,
+Respond_to_proposal,TRUE,
+Response,TRUE,
+Response_scenario,FALSE,
+Responsibility,FALSE,
+Rest,FALSE,
+Result_of_attempt_scenario,FALSE,
+Resurrection,TRUE,? similar to self_motion?
+Retaining,TRUE,
+Reveal_secret,TRUE,
+Revenge,TRUE,
+Revolution,FALSE,
+Rewards_and_punishments,TRUE,
+Ride_vehicle,TRUE,?
+Rising_to_a_challenge,TRUE,
+Risk_scenario,FALSE,
+Risky_situation,FALSE,
+Rite,TRUE,
+Roadways,FALSE,
+Robbery,TRUE,
+Rope_manipulation,TRUE,
+Rotting,FALSE,
+Run_risk,TRUE,
+Sacrificing_for,TRUE,
+Satisfying,TRUE,
+Scarcity,FALSE,
+Scheduling,TRUE,
+Scope,FALSE,
+Scouring,TRUE,
+Scrutinizing_for,FALSE,
+Scrutiny,TRUE,
+Searching_scenario,FALSE,
+Secrecy_status,FALSE,
+See_through,TRUE,
+Seeking,TRUE,
+Seeking_to_achieve,TRUE,
+Self_control,TRUE,
+Self_motion,TRUE,?
+Sending,TRUE,
+Sensation,FALSE,
+Sent_items,FALSE,
+Sentencing,TRUE,
+Separating,TRUE,
+Sequence,FALSE,
+Serving_in_capacity,TRUE,
+Set_of_interrelated_entities,FALSE,
+Set_relation,FALSE,
+Setting_back_burn,FALSE,
+Setting_fire,TRUE,
+Setting_out,TRUE,
+Severity_of_offense,FALSE,
+Sex,TRUE,
+Sexual_reproduction_scenario,FALSE,
+Shaped_part,FALSE,
+Shapes,FALSE,
+Sharing,TRUE,
+Sharpness,FALSE,
+Shoot_projectiles,TRUE,
+Shooting_scenario,FALSE,
+Shopping,TRUE,
+Short_selling,TRUE,
+Sidereal_appearance,FALSE,?
+Sign,TRUE,
+Sign_agreement,TRUE,
+Silencing,TRUE,
+Similarity,TRUE,? mimic etc
+Simple_name,FALSE,
+Simple_naming,TRUE,
+Simultaneity,FALSE,
+Size,FALSE,
+Sleep,TRUE,
+Sleep_wake_cycle,FALSE,
+Smuggling,TRUE,
+Soaking,TRUE,
+Soaking_up,TRUE,? metaphorical
+Sociability,FALSE,
+Social_behavior_evaluation,FALSE,
+Social_connection,FALSE,
+Social_desirability,FALSE,
+Social_event,FALSE,
+Social_event_collective,FALSE,
+Social_event_individuals,TRUE,
+Social_interaction_evaluation,FALSE,
+Socially_significant_history_scenario,FALSE,
+Sole_instance,FALSE,
+Sound_level,FALSE,
+Sound_movement,FALSE,?
+Sounds,FALSE,
+Source_of_getting,FALSE,
+Source_path_goal,FALSE,
+Spatial_co-location,FALSE,
+Spatial_contact,FALSE,
+Speak_on_topic,TRUE,
+Specific_individual,FALSE,
+Speed_description,FALSE,
+Spelling_and_pronouncing,TRUE,
+Sports_jargon,FALSE,
+Stage_of_progress,FALSE,
+Standing_by,TRUE,
+State,FALSE,
+State_continue,FALSE,
+State_of_entity,FALSE,
+Statement,TRUE,
+Stimulate_emotion,TRUE,? metaphorical
+Stimulus_focus,FALSE,
+Stinginess,TRUE,
+Store,FALSE,
+Storing,TRUE,
+Strictness,FALSE,
+Studying,TRUE,
+Suasion,TRUE,
+Subjective_influence,TRUE,
+Subjective_temperature,FALSE,?
+Submitting_documents,TRUE,
+Subordinates_and_superiors,TRUE,
+Subsisting,TRUE,
+Substance,FALSE,
+Substance_by_phase,FALSE,
+Subversion,TRUE,
+Success_or_failure,TRUE,
+Successful_action,TRUE,
+Successfully_communicate_message,TRUE,
+Sufficiency,FALSE,
+Suicide_attack,TRUE,? there are no verbs but there is an agentive example (‘they kamikazed the base’) 
+Suitability,FALSE,
+Summarizing,TRUE,
+Supply,TRUE,
+Supporting,TRUE,
+Surpassing,TRUE,metaphorical
+Surrendering,TRUE,
+Surrendering_possession,TRUE,
+Surrounding,TRUE,
+Surviving,TRUE,
+Suspicion,TRUE,
+Symmetrical_collective_reciprocality,FALSE,
+System,FALSE,
+System_complexity,FALSE,
+Take_place_of,TRUE,
+Taking,TRUE,
+Taking_captive,TRUE,
+Taking_sides,TRUE,
+Taking_time,TRUE,
+Talking_into,TRUE,
+Tasting,TRUE,
+Team,FALSE,
+Telling,TRUE,
+Temperature,FALSE,
+Temporal_collocation,FALSE,
+Temporal_pattern,FALSE,
+Temporal_subregion,FALSE,
+Temporary_group,FALSE,
+Temporary_leave,TRUE,
+Temporary_stay,TRUE,
+Temporary_transfer_scenario,FALSE,
+Terms_of_agreement,FALSE,
+Terrorism,FALSE,
+Text,FALSE,
+Text_creation,TRUE,
+Theft,TRUE,
+Thermodynamic_phase,FALSE,
+Thriving,TRUE,
+Thwarting,TRUE,
+Time_period_of_action,FALSE,
+Time_vector,FALSE,
+Timespan,FALSE,
+Timetable,FALSE,
+Tolerating,TRUE,
+Tool_purpose,FALSE,
+Topic,TRUE,
+Touring,TRUE,
+Toxic_substance,FALSE,
+Transfer,TRUE,
+Transfer_scenario,FALSE,
+Transition_to_a_quality,FALSE,
+Transition_to_a_situation,FALSE,
+Transition_to_a_state,FALSE,
+Transitive_action,FALSE,
+Translating,TRUE,
+Transportation_status,FALSE,
+Trap,FALSE,
+Travel,TRUE,
+Traversing,TRUE,
+Treating_and_mistreating,TRUE,
+Trendiness,FALSE,
+Trial,FALSE,
+Triggering,TRUE,
+Trust,TRUE,
+Try_defendant,TRUE,
+Trying_out,TRUE,
+Turning_out,FALSE,
+Type,FALSE,
+Typicality,FALSE,
+Unattributed_information,TRUE,'rumor.v’ only exists in passive form
+Undergo_change,FALSE,
+Undergo_transformation,FALSE,
+Undergoing,FALSE,
+Undergoing_scenario,FALSE,
+Undressing,TRUE,
+Unemployment_rate,FALSE,
+Use_firearm,TRUE,
+Use_vehicle,FALSE,
+Used_up,FALSE,
+Usefulness,TRUE,
+Using,TRUE,
+Using_resource,TRUE,
+Vehicle,FALSE,
+Vehicle_departure_initial_stage,TRUE,? metaphorical
+Vehicle_landing,TRUE,? metaphorical
+Vehicle_subpart,FALSE,
+Verdict,TRUE,
+Verification,TRUE,
+Version_sequence,FALSE,
+Victim_operated_IED,FALSE,
+Violence,FALSE,
+Visit_host,FALSE,empty
+Visit_host_arrival,FALSE,empty
+Visit_host_departure,FALSE,empty
+Visit_host_stay,FALSE,empty
+Visiting,TRUE,
+Visiting_scenario,FALSE,
+Visiting_scenario_arrival,FALSE,
+Visiting_scenario_departing,FALSE,
+Visiting_scenario_stay,FALSE,
+Visitor_and_host,FALSE,
+Visitor_arrival,FALSE,
+Visitor_departure,FALSE,
+Visitor_scenario,FALSE,
+Vocalizations,FALSE,
+Volubility,FALSE,
+Wagering,TRUE,
+Waiting,TRUE,
+Waking_up,TRUE,
+Want_suspect,FALSE,
+Warning,TRUE,
+Waver_between_options,TRUE,
+Wealthiness,FALSE,
+Weapon,FALSE,
+Wearing,TRUE,
+Weather,FALSE,
+Wholes_and_parts,FALSE,
+Willingness,FALSE,
+Win_prize,TRUE,
+Withdraw_from_participation,TRUE,
+Within_distance,FALSE,
+Word_relations,FALSE,
+Work,TRUE,
+Working_a_post,TRUE,
+Worry,TRUE,

resources/crashes_frame_list.txt

@@ -0,0 +1,14 @@
+# based on Gosse's presentation in 1st meeting with Marco
+Killing
+Death
+Impact
+Catch_fire
+Cause_harm
+
+# extra to make more comparable with femicide data
+Causation
+Cause_motion
+Dead_or_alive
+Emotion_directed
+Event
+Experience_bodily_harm

resources/crashes_frame_to_roles.csv

@@ -0,0 +1,11 @@
+frame,role:perpetrator_like,role:victim_like,role:cause_like,notes
+Catch_fire,-,-,-,
+Causation,Causer,Affected,Cause,
+Cause_harm,Agent,Victim,Cause,
+Cause_motion,-,-,-,does not seem to usually refer to the main murder event
+Dead_or_alive,-,Protagonist,Explanation,
+Death,-,Protagonist,Cause,
+Emotion_directed,-,-,-,does not seem to usually refer to the main murder event
+Event,-,-,-,does not involve any participants
+Experience_bodily_harm,Experiencer|Body_part,-,-,
+Killing,Killer,Victim,Cause,

resources/crashes_sources.csv

@@ -0,0 +1,440 @@
+ProviderName,ProviderFreq,ProviderNameCorr,RegionalScope,ContentType,MediumType,Country,Province,Locality,MediaCollection,MediaOwner,Duplicate,Notes
+noordhollandsdagblad.nl,620,Noordhollands Dagblad,Regional,GeneralNews,Newspaper,Netherlands,Noord-Holland,-,Mediahuis Nederland,Mediahuis,,
+gelderlander.nl,553,De Gelderlander,Regional,GeneralNews,Newspaper,Netherlands,Gelderland,-,ADR Nieuwsmedia,DPG Media,,
+nhnieuws.nl,479,NH Nieuws,Regional,GeneralNews,TV-Radio,Netherlands,Gelderland,-,-,[public],,
+rtvoost.nl,409,RTV Oost,Regional,GeneralNews,TV-Radio,Netherlands,Overijssel,-,-,[public],,
+www.ad.nl,399,Algemeen Dagblad,National,GeneralNews,Newspaper,Netherlands,-,-,ADR Nieuwsmedia,DPG Media,,
+rtvutrecht.nl,374,RTV Utrecht,National,GeneralNews,TV-Radio,Netherlands,Utrecht,-,-,[public],,
+destentor.nl,326,De Stentor,Regional,GeneralNews,Newspaper,Netherlands,Flevoland|Gelderland|Overijssel,-,ADR Nieuwsmedia,DPG Media,,
+omroepbrabant.nl,227,Omroep Brabant,Regional,GeneralNews,TV-Radio,Netherlands,Noord-Brabant,-,-,[public],,
+haarlemsdagblad.nl,219,Haarlems Dagblad,Local,GeneralNews,Newspaper,Netherlands,Noord-Holland,Haarlem,Mediahuis Nederland,Mediahuis,,
+www.ed.nl,206,Eindhovens Dagblad,Local,GeneralNews,Newspaper,Netherlands,Noord-Brabant,Eindhoven,ADR Nieuwsmedia,DPG Media,,
+www.bd.nl,198,Brabants Dagblad,Regional,GeneralNews,Newspaper,Netherlands,Noord-Brabant,-,ADR Nieuwsmedia,DPG Media,,
+weblogzwolle.nl,191,Weblog Zwolle,Local,GeneralNews,OnlineOnly,Netherlands,Overijssel,Zwolle,-,[independent],,
+www.at5.nl,173,AT5,Local,GeneralNews,TV-Radio,Netherlands,Noord-Holland,Amsterdam,-,[public],,
+rtvfocuszwolle.nl,168,RTV Focus Zwolle,Local,GeneralNews,TV-Radio,Netherlands,Overijssel,Zwolle,-,[independent],,
+hvzeeland.nl,163,HVZeeland.nl,Regional,EmergenciesNews,OnlineOnly,Netherlands,Zeeland,-,-,[independent],,
+omroepgelderland.nl,142,Omroep Brabant,Regional,GeneralNews,TV-Radio,Netherlands,Noord-Brabant,-,-,[public],,
+1Limburg | Nieuws en sport uit Limburg,135,1Limburg,Regional,GeneralNews,OnlineOnly,Netherlands,Limburg_NL,-,-,[public],,"belongs to ""1L"" tv/radio channel, but separate brand"
+www.hln.be,132,Het Laatste Nieuws,National,GeneralNews,Newspaper,Belgium,-,-,-,DPG Media,,
+telegraaf.nl,124,De Telegraaf,National,GeneralNews,Newspaper,Netherlands,-,-,Mediahuis Nederland,Mediahuis,,
+amstelveenz.nl,109,AmstelveenZ,Local,GeneralNews,Magazine,Netherlands,Noord-Holland,Amstelveen,-,[independent],,
+tubantia.nl,105,Tubantia,Regional,GeneralNews,Magazine,Netherlands,Overijssel,-,ADR Nieuwsmedia,DPG Media,,
+leidschdagblad.nl,100,Leidsch Dagblad,Local,GeneralNews,Magazine,Netherlands,Zuid-Holland,Leiden,-,DPG Media,,
+bndestem.nl,92,BN DeStem,Regional,GeneralNews,Newspaper,Netherlands,Noord-Brabant|Zeeland,-,ADR Nieuwsmedia,DPG Media,,
+nos.nl,92,NOS,National,GeneralNews,TV-Radio,Netherlands,-,-,NPO,[public],,
+hartvannederland.nl,90,Hart van Nederland,National,GeneralNews,TV-Radio,Netherlands,-,-,SBS6,Talpa Network,,
+Dagblad van het Noorden,85,Dagblad van het Noorden,Regional,GeneralNews,Newspaper,Netherlands,Drenthe|Groningen,-,NDC Mediagroep,Mediahuis,,
+rtvdrenthe.nl,84,RTV Drenthe,Regional,GeneralNews,TV-Radio,Netherlands,Drenthe,-,-,[public],,
+rtvnoord.nl,74,RTV Noord,Regional,GeneralNews,TV-Radio,Netherlands,Groningen,-,-,[public],,
+112groningen.nl,72,112Groningen.nl,Local,EmergenciesNews,OnlineOnly,Netherlands,Groningen,Groningen,-,[independent],,
+www.nu.nl,70,NU.nl ,National,GeneralNews,OnlineOnly,Netherlands,-,-,-,DPG Media,,
+omroepwest.nl,67,Omroep West,Regional,GeneralNews,TV-Radio,Netherlands,Zuid-Holland,-,-,[public],,
+RTV Rijnmond,62,RTV Rijnmond,Regional,GeneralNews,TV-Radio,Netherlands,Zuid-Holland,-,-,[public],,
+www.pzc.nl,62,Provinciale Zeeuwse Courant,Regional,GeneralNews,Newspaper,Netherlands,Zeeland,-,-,[public],,
+rijnmond.nl,61,RTV Rijnmond,Regional,GeneralNews,TV-Radio,Netherlands,Zuid-Holland,-,-,[public],"""RTV Rijnmond""",
+112Twente.nl,59,112Twente,Regional,GeneralNews,OnlineOnly,Netherlands,Overijssel,-,-,[independent],,
+Het Nieuwsblad,52,Het Nieuwsblad,National,GeneralNews,Newspaper,Belgium,-,-,-,[independent],,
+hbvl.be,50,Het Belang van Limburg,Regional,GeneralNews,Newspaper,Belgium,Limburg_BE,-,Mediahuis België,Mediahuis,,
+www.vrt.be,48,VRT,National,GeneralNews,TV-Radio,Belgium,-,-,-,[public],,
+De Limburger,47,De Limburger,Regional,GeneralNews,Newspaper,Netherlands,Limburg_NL,-,Mediahuis Limburg,Mediahuis,,
+112fryslan.nl,45,112 Fryslân,Regional,EmergenciesNews,OnlineOnly,Netherlands,Friesland,-,-,[independent],,
+112HM.nl,44,112HM.nl,Regional,EmergenciesNews,OnlineOnly,Netherlands,Zuid-Holland,-,-,[independent],,
+omroepzeeland.nl,43,Omroep Zeeland,Regional,GeneralNews,TV-Radio,Netherlands,Zeeland,-,-,[public],,
+GelreNieuws.nl,41,GelreNieuws,Regional,GeneralNews,OnlineOnly,Netherlands,Gelderland,-,-,Persbureau Heitink,,
+twitter.com,39,Twitter,International,SocialMedia,OnlineOnly,-,-,-,-,-,,
+Het Parool,39,Het Parool,National,GeneralNews,Newspaper,Netherlands,-,-,-,DPG Media,,strong local/regional focus but published nationally
+nieuwsblad.be,38,Het Nieuwsblad,National,GeneralNews,Newspaper,Belgium,-,-,-,[independent],"""Het Nieuwsblad""",
+dvhn.nl,33,Dagblad van het Noorden,Regional,GeneralNews,Newspaper,Netherlands,Drenthe|Groningen,-,NDC Mediagroep,Mediahuis,,
+politie.nl,33,Politie,National,OrganizationSpecific,OnlineOnly,Netherlands,-,-,-,-,,official website of the Dutch National Police
+112twente.nl,32,112Twente,Regional,GeneralNews,OnlineOnly,Netherlands,Overijssel,-,-,[independent],"""112Twente.nl""",
+hardnieuws.nl,32,Hardnieuws,National,EmergenciesNews,OnlineOnly,Netherlands,-,-,-,[independent],,
+112 Overijssel,32,112 Overijssel,Regional,EmergenciesNews,OnlineOnly,Netherlands,Overijssel,-,-,[independent],,
+www.lc.nl,30,Leeuwarder Courant,Regional,GeneralNews,Newspaper,Netherlands,Friesland,-,-,[independent],,
+rtlnieuws.nl,29,RTL Nieuws,National,GeneralNews,TV-Radio,Netherlands,-,-,RTL Nederland,RTL Group,,
+rtva.nl,28,RTVA,Local,GeneralNews,TV-Radio,Netherlands,Noord-Holland,Amstelveen,-,[independent],,
+Leeuwarder Courant,28,Leeuwarder Courant,Regional,GeneralNews,Newspaper,Netherlands,Friesland,-,-,[independent],"""www.lc.nl""",
+Gazet van Antwerpen,26,Gazet van Antwerpen,Regional,GeneralNews,Newspaper,Belgium,Antwerpen,-,Mediahuis België,Mediahuis,,
+District8.net,26,District8,Regional,GeneralNews,OnlineOnly,Netherlands,Zuid-Holland,-,-,[independent],,
+Focus en WTV,24,Focus-WTV,Regional,GeneralNews,TV-Radio,Belgium,West-Vlaanderen,-,-,Roularta Media Group,,
+studio040.nl,24,Studio 040,Local,GeneralNews,TV-Radio,Netherlands,Noord-Brabant,Eindhoven,-,[independent],,
+112-Overijssel,24,112 Overijssel,Regional,EmergenciesNews,OnlineOnly,Netherlands,Overijssel,-,-,[independent],,
+omroepflevoland.nl,23,Omroep Flevoland,Regional,GeneralNews,TV-Radio,Netherlands,Flevoland,-,-,[public],,
+De Utrechtse Internet Courant,20,De Utrechtse Internet Courant,Local,GeneralNews,OnlineOnly,Netherlands,Utrecht,Utrecht,-,[independent],,
+www.wos.nl,19,WOS,Local,GeneralNews,TV-Radio,Netherlands,Zuid-Holland,Maassluis,-,[independent],,
+wos.nl,19,WOS,Local,GeneralNews,TV-Radio,Netherlands,Zuid-Holland,Maassluis,-,[independent],"""www.wos.nl""",
+OOG Radio en Televisie,18,OOG,Local,GeneralNews,TV-Radio,Netherlands,Groningen,Groningen,-,[independent],,
+112barneveld.nl,17,112 Barneveld,Local,EmergenciesNews,TV-Radio,Netherlands,Gelderland,Barneveld,-,112Press,,
+112hm.nl,17,112HM.nl,Regional,EmergenciesNews,OnlineOnly,Netherlands,Zuid-Holland,-,-,[independent],,
+flashphoto.nl,17,FlashPhoto,Local,EmergenciesNews,OnlineOnly,Netherlands,Zuid-Holland,Rotterdam,-,[independent],,"specialized in photography, also (emergency) news"
+TVOOST - Regionaal nieuws uit Oost-Vlaanderen,16,TV Oost,Regional,GeneralNews,TV-Radio,Belgium,Oost-Vlaanderen,-,-,Concentra,,
+zwollenu.nl,15,ZwolleNu,Local,GeneralNews,OnlineOnly,Netherlands,Overijssel,Zwolle,-,[independent],,
+112ede.nl,15,112 Ede,Local,EmergenciesNews,OnlineOnly,Netherlands,Gelderland,Ede,-,112Press,,
+112brabant.nl,13,112 Brabant,Regional,EmergenciesNews,OnlineOnly,Netherlands,Noord-Brabant,-,-,[independent],,
+TVL - Dagelijks nieuws uit Limburg,13,TVL,Regional,GeneralNews,TV-Radio,Belgium,Limburg_BE,-,-,Concentra,,
+oogtv.nl,13,OOG,Local,GeneralNews,TV-Radio,Netherlands,Groningen,Groningen,-,[independent],"""OOG""",
+zhzactueel.nl,12,ZHZ Actueel,Regional,EmergenciesNews,TV-Radio,Netherlands,Zuid-Holland,-,-,[independent],,
+www.nrc.nl,12,NRC,National,GeneralNews,Newspaper,Netherlands,-,-,NRC Media,Mediahuis,,
+stedendriehoek.net,12,Nieuwsblad Stedendriehoek,Regional,GeneralNews,Newspaper,Netherlands,Gelderland|Overijssel,-,-,[independent],,
+ijmuidercourant.nl,11,IJmuider Courant,Local,GeneralNews,Newspaper,Netherlands,Noord-Holland,IJmuiden,Mediahuis Nederland,Mediahuis,,
+Meternieuws.nl,10,Meter Nieuws,Regional,EmergenciesNews,OnlineOnly,Netherlands,Drenthe|Groningen|Overijssel,-,-,[independent],,
+deswollenaer.nl,10,De Swollenaer,Local,GeneralNews,Newspaper,Netherlands,Overijssel,Zwolle,-,Brug Media,,
+alkmaarcentraal.nl,10,Alkmaar Centraal,Local,GeneralNews,OnlineOnly,Netherlands,Noord-Holland,Alkmaar,-,[independent],,
+112Vandaag,10,112Vandaag,National,EmergenciesNews,OnlineOnly,Netherlands,-,-,-,[independent],,
+mediatv.nl,10,MediaTV,National,EmergenciesNews,OnlineOnly,Netherlands,-,-,-,[independent],,
+"gelderlander.nl, het laatste nieuws uit binnen- en buitenland, sport en show",10,De Gelderlander,Regional,GeneralNews,Newspaper,Netherlands,Gelderland,-,ADR Nieuwsmedia,DPG Media,"""gelderlander.nl""",
+Weertdegekste.nl,9,,,,,,,,,,,
+WâldNet,9,,,,,,,,,,,
+transport-online.nl,9,,,,,,,,,,,
+noordernieuws.nl,9,,,,,,,,,,,
+regiopurmerend.nl,8,,,,,,,,,,,
+https://www.vlaardingen24.nl,8,,,,,,,,,,,
+Groninger Gezinsbode,8,,,,,,,,,,,
+"Ring TV | Jouw zender, Jouw nieuws",8,,,,,,,,,,,
+blikopnieuws.nl,8,,,,,,,,,,,
+edestad.nl,8,,,,,,,,,,,
+steenwijkercourant.nl,8,,,,,,,,,,,
+nieuwsopbeeld.nl,8,,,,,,,,,,,
+ROB-tv - Regionale Omroep Brabant,8,,,,,,,,,,,
+barneveldsekrant.nl,8,,,,,,,,,,,
+https://www.schiedam24.nl,8,,,,,,,,,,,
+Sleutelstad.nl,7,,,,,,,,,,,
+Unity NU is de nieuwssite voor de regio Leiden [www.unity.nu],7,,,,,,,,,,,
+112WestFriesland.nl,7,,,,,,,,,,,
+112vallei.nl,7,,,,,,,,,,,
+Omroep Gelderland,7,,,,,,,,,,,
+Het Belang van Limburg,7,,,,,,,,,,,
+sleutelstad.nl,7,,,,,,,,,,,
+Bredavandaag|HétnieuwsuitBreda,6,,,,,,,,,,,
+alarmeringen.nl,6,,,,,,,,,,,
+stedendriehoek.nl,6,,,,,,,,,,,
+halstadcentraal.nl,6,,,,,,,,,,,
+Westlanders.nu,6,,,,,,,,,,,
+ATV - Antwerpse televisie,6,,,,,,,,,,,
+Stefan Verkerk Fotografie en Webdesign,6,,,,,,,,,,,
+De Gooi- en Eemlander,6,,,,,,,,,,,
+alphens.nl,6,,,,,,,,,,,
+112nieuwsonline.nl,6,,,,,,,,,,,
+zwollezuidnieuws.nl,6,,,,,,,,,,,
+1Limburg,5,,,,,,,,,,,
+denoordoostpolder.nl,5,,,,,,,,,,,
+112provincieutrecht.nl,5,,,,,,,,,,,
+rtvzaanstreek.nl,5,,,,,,,,,,,
+nederweert24.nl,5,,,,,,,,,,,
+Nieuws dat je raakt. 24/24u – Nnieuws.be,5,,,,,,,,,,,
+nieuws.nl,5,,,,,,,,,,,
+RTV Oost,5,,,,,,,,,,,
+regio15.nl,5,,,,,,,,,,,
+De Standaard,5,,,,,,,,,,,
+flevopost.nl,5,,,,,,,,,,,
+regionieuwshoogeveen.nl,5,,,,,,,,,,,
+petershotnews.nl | Nieuws & fotografie,5,,,,,,,,,,,
+Nieuws op Beeld - Altijd het laatste (112) nieuws vanuit de regio Rotterdam-Rijnmond!,5,,,,,,,,,,,
+ZwolleZuidNieuws: alles wat Zwolle Zuid beweegt!,4,,,,,,,,,,,
+Telegraaf,4,,,,,,,,,,,
+RTV Utrecht,4,,,,,,,,,,,
+regioleidscherijn.nl,4,,,,,,,,,,,
+Hart van Nederland,4,,,,,,,,,,,
+dagblad070.nl,4,,,,,,,,,,,
+nuus.be,4,,,,,,,,,,,
+onswestfriesland.nl,4,,,,,,,,,,,
+waldnet.nl,4,,,,,,,,,,,
+NU,4,,,,,,,,,,,
+www.gva.be,4,,,,,,,,,,,
+bunniksnieuws.nl,4,,,,,,,,,,,
+dalfsennet.nl,4,,,,,,,,,,,
+112heuvelrug.nl,4,,,,,,,,,,,
+hartvanlansingerland.nl,4,,,,,,,,,,,
+"AD.nl, het laatste nieuws uit binnen- en buitenland, sport en show",4,,,,,,,,,,,
+bruzz.be,4,,,,,,,,,,,
+Vlissingen-Internetbode,3,,,,,,,,,,,
+Blik op nieuws,3,,,,,,,,,,,
+limburg24.nl,3,,,,,,,,,,,
+www.gld.nl,3,,,,,,,,,,,
+112zwolle.nl,3,,,,,,,,,,,
+omroepvenray.nl,3,,,,,,,,,,,
+lokaalgelderland.nl,3,,,,,,,,,,,
+destadgorinchem.nl,3,,,,,,,,,,,
+112veenendaal.nl,3,,,,,,,,,,,
+denhaagfm.nl,3,,,,,,,,,,,
+facebook.com,3,,,,,,,,,,,
+112midden-zeeland.nl,3,,,,,,,,,,,
+de Volkskrant,3,,,,,,,,,,,
+meppelercourant.nl,3,,,,,,,,,,,
+Neustadt-Geflüster,3,,,,,,,,,,,
+goudsdagblad.nl,3,,,,,,,,,,,
+schie.nu,3,,,,,,,,,,,
+oozo.nl,3,,,,,,,,,,,
+www.rd.nl,3,,,,,,,,,,,
+voorburgsdagblad.nl,3,,,,,,,,,,,
+NieuwsOverijssel.nl,3,,,,,,,,,,,
+ZwolleZuidNieuws: alles wat Zwolle-Zuid beweegt!,3,,,,,,,,,,,
+112inbeeld.nl,3,,,,,,,,,,,
+bredavandaag.nl,3,,,,,,,,,,,
+De Jutter | De Hofgeest,2,,,,,,,,,,,
+Woerden.TV,2,,,,,,,,,,,
+knipselkrant-curacao.com,2,,,,,,,,,,,
+heerenveensecourant.nl,2,,,,,,,,,,,
+ThePostOnline,2,,,,,,,,,,,
+regio8.nl,2,,,,,,,,,,,
+BarendrechtNU.nl,2,,,,,,,,,,,
+"pzc.nl, het laatste nieuws uit binnen- en buitenland, sport en show",2,,,,,,,,,,,
+weespernieuws.nl,2,,,,,,,,,,,
+Amstelveenz,2,,,,,,,,,,,
+stadtiel.nl,2,,,,,,,,,,,
+gouweijsselnieuws.nl,2,,,,,,,,,,,
+Nieuws op Beeld,2,,,,,,,,,,,
+heerhugowaardcentraal.nl,2,,,,,,,,,,,
+nieuwsbladdezaankanter.nl,2,,,,,,,,,,,
+www.avs.be,2,,,,,,,,,,,
+haarlemsweekblad.nl,2,,,,,,,,,,,
+yomyom.net,2,,,,,,,,,,,
+mooirooi.nl,2,,,,,,,,,,,
+oisterwijknieuws.nl,2,,,,,,,,,,,
+rtv-apeldoorn.nl,2,,,,,,,,,,,
+112amersfoort.nl,2,,,,,,,,,,,
+dedemsvaartsecourant.nl,2,,,,,,,,,,,
+ed.nl,2,,,,,,,,,,,
+soestercourant.nl,2,,,,,,,,,,,
+heemsteedsecourant.nl,2,,,,,,,,,,,
+112hoogezand.nl,2,,,,,,,,,,,
+hetstreekblad.nl,2,,,,,,,,,,,
+NRC,2,,,,,,,,,,,
+112nieuws.net,2,,,,,,,,,,,
+De Limburger Mobile,2,,,,,,,,,,,
+0297.nl,2,,,,,,,,,,,
+drachtstercourant.nl,2,,,,,,,,,,,
+Sittard-Geleen,2,,,,,,,,,,,
+hoogenlaag.nl,2,,,,,,,,,,,
+drentsnieuws.nl,2,,,,,,,,,,,
+brugnieuws.nl,2,,,,,,,,,,,
+medemblikactueel.nl,2,,,,,,,,,,,
+rechtspraak.nl,2,,,,,,,,,,,
+gooieneembode.nl,2,,,,,,,,,,,
+arenalokaal.nl,2,,,,,,,,,,,
+DitisdeZaanstreek.nl,2,,,,,,,,,,,
+hcnieuws.nl,2,,,,,,,,,,,
+https://www.heerhugowaardsdagblad.nl/,2,,,,,,,,,,,
+schagenfm.nl,2,,,,,,,,,,,
+hv-almere.nl,2,,,,,,,,,,,
+112achterhoek-nieuws.nl,2,,,,,,,,,,,
+peelenmaasvenray.nl,2,,,,,,,,,,,
+frieslandactueel.nl,2,,,,,,,,,,,
+www.rtv.be,2,,,,,,,,,,,
+hoogeveenschecourant.nl,2,,,,,,,,,,,
+Nieuws Apeldoorn Direct,2,,,,,,,,,,,
+nieuwsuitberkelland.nl,2,,,,,,,,,,,
+112meerlanden.nl,2,,,,,,,,,,,
+internetbode.nl,2,,,,,,,,,,,
+nieuw-volendam.nl,2,,,,,,,,,,,
+katwijkactueel.nl,2,,,,,,,,,,,
+112schiedam.nl,2,,,,,,,,,,,
+compactmedia.nl,2,,,,,,,,,,,
+culemborgsecourant.nl,2,,,,,,,,,,,
+Alphens.nl,2,,,,,,,,,,,
+112ijmond.nl,2,,,,,,,,,,,
+detoren.net,2,,,,,,,,,,,
+gorkumsnieuws.nl,2,,,,,,,,,,,
+Redactie24.be,2,,,,,,,,,,,
+wnl.tv,2,,,,,,,,,,,
+alarmeringdroid.nl,1,,,,,,,,,,,
+HCNieuws,1,,,,,,,,,,,
+frontpage.fok.nl,1,,,,,,,,,,,
+112vdg.nl,1,,,,,,,,,,,
+Ede Stad,1,,,,,,,,,,,
+my net rosh haayin,1,,,,,,,,,,,
+Noordhollands Dagblad,1,,,,,,,,,,,
+Zundert-Internetbode,1,,,,,,,,,,,
+defeanster.nl,1,,,,,,,,,,,
+heerhugowaardalife.nl,1,,,,,,,,,,,
+inteylingen.nl,1,,,,,,,,,,,
+The News Herald,1,,,,,,,,,,,
+Rijswijk.TV,1,,,,,,,,,,,
+Leidsch Dagblad,1,,,,,,,,,,,
+mynetkrayot,1,,,,,,,,,,,
+OldambtNu.nl,1,,,,,,,,,,,
+instagram.com,1,,,,,,,,,,,
+Bonaire.Nu,1,,,,,,,,,,,
+nieuwsbladdekoerier.nl,1,,,,,,,,,,,
+BergenopZoom-Internetbode,1,,,,,,,,,,,
+1twente.nl,1,,,,,,,,,,,
+www.rtl.de,1,,,,,,,,,,,
+tvvalkenburg.tv,1,,,,,,,,,,,
+alarmfase1.nl,1,,,,,,,,,,,
+gids.tv,1,,,,,,,,,,,
+RTV Uitgeest,1,,,,,,,,,,,
+De Telegraaf,1,,,,,,,,,,,
+112-dokkum.nl,1,,,,,,,,,,,
+wijksnieuws.nl,1,,,,,,,,,,,
+hetkontakt.nl,1,,,,,,,,,,,
+landelijkeorganisatieverkeersslachtoffers.nl,1,,,,,,,,,,,
+rtv.be,1,,,,,,,,,,,
+indebuurt Ede,1,,,,,,,,,,,
+112 groningen.nl,1,,,,,,,,,,,
+Ik hou van Arnhem,1,,,,,,,,,,,
+112hardenberg.nu,1,,,,,,,,,,,
+stadwageningen.nl,1,,,,,,,,,,,
+ridderkerksdagblad.nl,1,,,,,,,,,,,
+geenstijl.nl,1,,,,,,,,,,,
+dewoudenberger.nl,1,,,,,,,,,,,
+https://www.alkmaarsdagblad.nl/,1,,,,,,,,,,,
+nieuwsbladnof.nl,1,,,,,,,,,,,
+Nieuwe Meerbode,1,,,,,,,,,,,
+looopings.nl,1,,,,,,,,,,,
+amstelveensnieuwsblad.nl,1,,,,,,,,,,,
+texelsecourant.nl,1,,,,,,,,,,,
+anwb.nl,1,,,,,,,,,,,
+indebuurt Delft,1,,,,,,,,,,,
+https://www.zutphen24.nl,1,,,,,,,,,,,
+Teylingen,1,,,,,,,,,,,
+112Midden-Zeeland,1,,,,,,,,,,,
+noorderkrant.nl,1,,,,,,,,,,,
+onswestbrabant.nl,1,,,,,,,,,,,
+lindanieuws.nl,1,,,,,,,,,,,
+112persfotografie.nl,1,,,,,,,,,,,
+antilliaansdagblad.com,1,,,,,,,,,,,
+Site-Knack-NL,1,,,,,,,,,,,
+alblasserdamsnieuws.nl,1,,,,,,,,,,,
+112harderwijk.nl,1,,,,,,,,,,,
+l1.nl,1,,,,,,,,,,,
+Nederweert24,1,,,,,,,,,,,
+Radio.NL,1,,,,,,,,,,,
+LokaalGelderland,1,,,,,,,,,,,
+hoekschnieuws.nl,1,,,,,,,,,,,
+nieuwsbladgeldermalsen.nl,1,,,,,,,,,,,
+Veenendaalse Krant,1,,,,,,,,,,,
+112-nederland.nl,1,,,,,,,,,,,
+demorgen.be,1,,,,,,,,,,,
+www.gic.nl,1,,,,,,,,,,,
+Unity NU is de nieuwssite voor de regio Leiden,1,,,,,,,,,,,
+Middelburg-Internetbode,1,,,,,,,,,,,
+groot-waterland.nl,1,,,,,,,,,,,
+regiobodeonline.nl,1,,,,,,,,,,,
+Nudrenthe.nl | Boven op het Nieuws |,1,,,,,,,,,,,
+Gocar.be,1,,,,,,,,,,,
+KW.be - Nieuws uit West-Vlaanderen,1,,,,,,,,,,,
+harenerweekblad.nl,1,,,,,,,,,,,
+nbcnews.com,1,,,,,,,,,,,
+Omroep Brabant,1,,,,,,,,,,,
+112apeldoorn.nl,1,,,,,,,,,,,
+linda.nl,1,,,,,,,,,,,
+assercourant.nl,1,,,,,,,,,,,
+prorail.nl,1,,,,,,,,,,,
+bbc.co.uk,1,,,,,,,,,,,
+schipholregio.nl,1,,,,,,,,,,,
+lequipe.fr,1,,,,,,,,,,,
+Politie.nl,1,,,,,,,,,,,
+welingelichtekringen.nl,1,,,,,,,,,,,
+destadamersfoort.nl,1,,,,,,,,,,,
+curacaonieuws.nu,1,,,,,,,,,,,
+Incidenten Apeldoorn e.o.,1,,,,,,,,,,,
+arubanieuws.nu,1,,,,,,,,,,,
+Vrij Nederland,1,,,,,,,,,,,
+Omroep Brabant,1,,,,,,,,,,,
+hetdeventernieuws.nl,1,,,,,,,,,,,
+Krimpenerwaard,1,,,,,,,,,,,
+avrotros.nl,1,,,,,,,,,,,
+elpais.com.co,1,,,,,,,,,,,
+112marum.nl,1,,,,,,,,,,,
+https://www.denheldersdagblad.nl/,1,,,,,,,,,,,
+ZHZActueel,1,,,,,,,,,,,
+bashinform.ru,1,,,,,,,,,,,
+FOK!,1,,,,,,,,,,,
+bx1.be,1,,,,,,,,,,,
+denhelderactueel.nl,1,,,,,,,,,,,
+www.bbc.com,1,,,,,,,,,,,
+eemskrant.nl,1,,,,,,,,,,,
+Regio Leidsche Rijn,1,,,,,,,,,,,
+Omroep Zeeland,1,,,,,,,,,,,
+topics.nl,1,,,,,,,,,,,
+HetKrantje-Online.nl,1,,,,,,,,,,,
+https://www.langedijkerdagblad.nl/,1,,,,,,,,,,,
+SpoorPro.nl,1,,,,,,,,,,,
+radio2.be,1,,,,,,,,,,,
+Metronieuws.nl,1,,,,,,,,,,,
+caribischnetwerk.ntr.nl,1,,,,,,,,,,,
+het-westerkwartier.nl,1,,,,,,,,,,,
+rijschoolpro.nl,1,,,,,,,,,,,
+rn7.nl,1,,,,,,,,,,,
+Eemskrant,1,,,,,,,,,,,
+HS-Krant,1,,,,,,,,,,,
+grootheerenveen.nl,1,,,,,,,,,,,
+RTV Zaanstreek,1,,,,,,,,,,,
+joustercourant.nl,1,,,,,,,,,,,
+112vlissingen-souburg.nl,1,,,,,,,,,,,
+112 Groningen,1,,,,,,,,,,,
+ZuidOosthoeker,1,,,,,,,,,,,
+AD.nl,1,,,,,,,,,,,
+Eemskrant | Nieuws uit de regio,1,,,,,,,,,,,
+Steenwijkerland,1,,,,,,,,,,,
+112tv.nl,1,,,,,,,,,,,
+Groningen,1,,,,,,,,,,,
+Reno Gazette Journal,1,,,,,,,,,,,
+haspengouwsnieuws.be,1,,,,,,,,,,,
+stellingwerf.nl,1,,,,,,,,,,,
+globo.com,1,,,,,,,,,,,
+112lansingerland.nu,1,,,,,,,,,,,
+bicycling.com,1,,,,,,,,,,,
+woldercourant.nl,1,,,,,,,,,,,
+omroepalmere.nl,1,,,,,,,,,,,
+Den Helder actueel,1,,,,,,,,,,,
+rtvhattem.nl,1,,,,,,,,,,,
+WNL,1,,,,,,,,,,,
+Omroep Venray,1,,,,,,,,,,,
+Dagblad070,1,,,,,,,,,,,
+friesenieuwsflitsen.nl,1,,,,,,,,,,,
+Kampen Online,1,,,,,,,,,,,
+dailymail.co.uk,1,,,,,,,,,,,
+https://112hm.nl/2021/07/21/ernstig-ongeval-op-de-hoogeveenseweg-hazerswoude-dorp-veroorzaakt-door-overstekende-hond/,1,,,,,,,,,,,
+112insteenwijkerland.nl,1,,,,,,,,,,,
+varnws.nl,1,,,,,,,,,,,
+actu.fr,1,,,,,,,,,,,
+hetkompashardinxveld-giessendam.nl,1,,,,,,,,,,,
+uitkijkpost.nl,1,,,,,,,,,,,
+RN7,1,,,,,,,,,,,
+NOS,1,,,,,,,,,,,
+uitzendinggemist.net,1,,,,,,,,,,,
+Nachrichten aus Leipzig - Leipziger Zeitung,1,,,,,,,,,,,
+twentefm.nl,1,,,,,,,,,,,
+Sergevanduijnhoven's Blog,1,,,,,,,,,,,
+Barneveldse Krant,1,,,,,,,,,,,
+leuvenactueel.be,1,,,,,,,,,,,
+https://www.schagerdagblad.nl/,1,,,,,,,,,,,
+coevordenhuisaanhuis.nl,1,,,,,,,,,,,
+blinker.co.il,1,,,,,,,,,,,
+Genderendigitaal,1,,,,,,,,,,,
+De Gelderlander,1,,,,,,,,,,,
+dagblad010.nl,1,,,,,,,,,,,
+traumaheli-mmt.nl,1,,,,,,,,,,,
+limburger.nl,1,,,,,,,,,,,
+Roosendaal-Internetbode,1,,,,,,,,,,,
+bommelerwaardgids.nl,1,,,,,,,,,,,
+Alkmaar Centraal,1,,,,,,,,,,,
+IJsselmondeNieuws en omstreken op facebook.com,1,,,,,,,,,,,
+theguardian.com,1,,,,,,,,,,,
+112 Vlissingen & Souburg,1,,,,,,,,,,,
+rtvpurmerend.nl,1,,,,,,,,,,,
+Site-KW-NL,1,,,,,,,,,,,
+10yan.com,1,,,,,,,,,,,
+petershotnews.nl,1,,,,,,,,,,,
+Dumbarton and Vale of Leven Reporter,1,,,,,,,,,,,
+cyclingweekly.com,1,,,,,,,,,,,
+hanzestad.nl,1,,,,,,,,,,,
+emmen.nu,1,,,,,,,,,,,
+foxreno.com,1,,,,,,,,,,,
+De Krant van Midden-Drenthe,1,,,,,,,,,,,
+BBC,1,,,,,,,,,,,
+112drachten.nl,1,,,,,,,,,,,
+brummensnieuws.nl,1,,,,,,,,,,,
+Streetsblog New York City,1,,,,,,,,,,,
+De Heemsteder,1,,,,,,,,,,,
+indebuurt Utrecht,1,,,,,,,,,,,
+westfriesweekblad.nl,1,,,,,,,,,,,
+1istochnik.ru,1,,,,,,,,,,,
+kipa.co.il,1,,,,,,,,,,,
+veluweland.nl,1,,,,,,,,,,,
+DNN - Dresdner Neueste Nachrichten,1,,,,,,,,,,,
+112wijchensnieuws.nl,1,,,,,,,,,,,
+delpher.nl,1,,,,,,,,,,,
+indebuurt Doetinchem,1,,,,,,,,,,,
+news4jax.com,1,,,,,,,,,,,

resources/dep_labels.txt

@@ -0,0 +1,159 @@
+acl:relcl↑
+acl:relcl↓
+acl↑
+acl↓
+advcl↑
+advcl↓
+advmod↑
+advmod↓
+amod↑
+amod↓
+appos↑
+appos↓
+aux:pass↑
+aux:pass↓
+aux↑
+aux↓
+case↑
+case↓
+ccomp↑
+ccomp↓
+cc↑
+cc↓
+compound:prt↑
+compound:prt↓
+compound↑
+compound↓
+conj↑
+conj↓
+cop↑
+cop↓
+csubj↑
+csubj↓
+dep↑
+dep↓
+det:poss↑
+det:poss↓
+det:predet↑
+det:predet↓
+det↑
+det↓
+discourse↑
+discourse↓
+expl:impers↑
+expl:impers↓
+expl:pass↓
+expl:pv↓
+expl↑
+expl↓
+fixed↑
+fixed↓
+flat:foreign↑
+flat:name↑
+flat:name↓
+flat↑
+flat↓
+iobj↑
+iobj↓
+mark↑
+mark↓
+nmod:poss↑
+nmod:poss↓
+nmod↑
+nmod↓
+nsubj:pass↑
+nsubj:pass↓
+nsubj↑
+nsubj↓
+nummod↑
+nummod↓
+obj↑
+obj↓
+obl:agent↑
+obl:agent↓
+obl↑
+obl↓
+orphan↓
+parataxis↑
+parataxis↓
+punct↑
+punct↓
+vocative↑
+vocative↓
+xcomp↑
+xcomp↓
+↑--acl:relcl↓
+↑--acl↓
+↑--advcl↓
+↑--advmod↓
+↑--amod↓
+↑--appos↓
+↑--aux:pass↓
+↑--aux↓
+↑--case↓
+↑--ccomp↓
+↑--cc↓
+↑--compound:prt↓
+↑--compound↓
+↑--conj↓
+↑--cop↓
+↑--csubj↓
+↑--dep↓
+↑--det:poss↓
+↑--det↓
+↑--discourse↓
+↑--expl:impers↓
+↑--expl:pass↓
+↑--expl↓
+↑--fixed↓
+↑--flat:foreign↓
+↑--flat:name↓
+↑--flat↓
+↑--iobj↓
+↑--mark↓
+↑--nmod:poss↓
+↑--nmod↓
+↑--nsubj:pass↓
+↑--nsubj↓
+↑--nummod↓
+↑--obj↓
+↑--obl:agent↓
+↑--obl↓
+↑--parataxis↓
+↑--xcomp↓
+↓--acl:relcl↓
+↓--acl↓
+↓--advcl↓
+↓--advmod↓
+↓--amod↓
+↓--appos↓
+↓--aux:pass↓
+↓--aux↓
+↓--case↓
+↓--ccomp↓
+↓--cc↓
+↓--compound:prt↓
+↓--compound↓
+↓--conj↓
+↓--cop↓
+↓--dep↓
+↓--det:poss↓
+↓--det↓
+↓--expl:impers↓
+↓--expl↓
+↓--fixed↓
+↓--flat:name↓
+↓--flat↓
+↓--iobj↓
+↓--mark↓
+↓--nmod:poss↓
+↓--nmod↓
+↓--nsubj:pass↓
+↓--nsubj↓
+↓--nummod↓
+↓--obj↓
+↓--obl:agent↓
+↓--obl↓
+↓--parataxis↓
+↓--xcomp↓
+⋆

resources/femicide_frame_list.txt

@@ -0,0 +1,23 @@
+# 1) based on Gaetana & Marion's template file
+Abusing#Violence
+Attack#Violence
+Cause_harm#Violence
+Hit_target#Violence
+Killing#Murder
+Rape#Violence
+Use_firearm#Violence
+
+# 2) based on Gosse's paper/document (Table 2)
+Attack#Violence
+Causation
+Cause_harm#Violence
+Cause_motion
+Emotion_directed
+Event#Murder
+Quarreling
+
+Dead_or_alive#Murder
+Death#Murder
+Experience_bodily_harm
+Killing#Murder
+Catastrophe#Murder

resources/femicides_frame_to_roles.csv

@@ -0,0 +1,16 @@
+frame,role:perpetrator_like,role:victim_like,role:cause_like,notes
+Abusing,Abuser,Victim,-,
+Attack,Assailant,Victim,-,
+Causation,Causer,Affected,Cause,
+Cause_harm,Agent,Victim,Cause,
+Cause_motion,-,-,-,does not seem to usually refer to the main murder event
+Dead_or_alive,-,Protagonist,Explanation,
+Death,-,Protagonist,Cause,
+Emotion_directed,-,-,-,does not seem to usually refer to the main murder event
+Event,-,-,-,does not involve any participants
+Experience_bodily_harm,Experiencer|Body_part,-,-,
+Hit_target,Agent,Target,-,
+Killing,Killer,Victim,Cause,
+Quarreling,-,-,-,core roles (Arguers/Arguer1/Arguer2) could denote either Perpetrator or victim
+Rape,Perpetrator,Victim,-,
+Use_firearm,Agent,Goal,-,

resources/migration_frame_list.txt

@@ -0,0 +1,56 @@
+ARRIVING#Travel
+CAUSE_MOTION#Travel
+DEPARTING#Travel
+FLUIDIC_MOTION#Travel
+SELF_MOTION#Travel
+TRAVEL#Travel
+DISEMBARKING#Travel
+RISKY_SITUATION#Travel
+DEATH#Travel
+
+CARDINAL_NUMBERS#Quantification
+CHANGE_OF_QUANTITY_OF_POSSESSION#Quantification
+CHANGE_POSITION_ON_A_SCALE#Quantification
+FAMILIARITY#Quantification
+INCREMENT#Quantification
+PROLIFERATING_IN_NUMBER#Quantification
+QUANTIFIED_MASS#Quantification
+QUANTITY#Quantification
+
+ABUSING#Crime
+ARREST#Crime
+COMMITTING_CRIME#Crime
+INTENTIONAL_DECEPTION#Crime
+KILLING#Crime
+RAPE#Crime
+ROBBERY#Crime
+SMUGGLING#Crime
+PROTEST#Crime
+THEFT#Crime
+CAUSE_HARM#Crime
+
+HOSTILE_ENCOUNTER#Hostility
+INVADING#Hostility
+ATTACK#Hostility
+WEAPON#Hostility
+
+ARRANGING#Administration
+MAKING_ARRANGEMENTS#Administration
+DISCUSSION#Administration
+EXECUTE_PLAN#Administration
+LEADERSHIP#Administration
+EXPEND_RESOURCE#Administration
+GATHERING_UP#Administration
+PLACING#Administration
+POINT_OF_DISPUTE#Administration
+INHIBIT_MOVEMENT#Administration
+EXPENSIVENESS#Administration
+
+ASSISTANCE#Humanizing
+HIRING#Humanizing
+INTENTIONALLY_CREATE#Humanizing
+SOCIAL_EVENT#Humanizing
+KINSHIP#Humanizing
+COLLABORATION#Humanizing
+EDUCATION_TEACHING#Humanizing
+RESCUING#Humanizing

sociofillmore/common/analyze_text.py

@@ -0,0 +1,1046 @@
+import io
+import json
+import os
+import sys
+import argparse
+import re
+import tarfile
+from collections import defaultdict
+import dataclasses
+from datetime import datetime
+from typing import Any, Dict, List, Tuple, Optional
+
+import pandas as pd
+import spacy
+from nltk.corpus import framenet as fn
+from nltk.corpus.reader.framenet import FramenetError
+from spacy.tokens import Token
+
+from sociofillmore.crashes.utils import is_a_dutch_text
+
+ITALIAN_ACTIVE_AUX = ["avere", "ha", "ho", "hai", "avete", "hanno", "abbiamo"]
+DUTCH_ACTIVE_AUX = ["heb", "hebben", "heeft"]
+
+active_frames_df = pd.read_csv("resources/active_frames_full.csv")
+ACTIVE_FRAMES = active_frames_df[active_frames_df["active"]]["frame"].tolist()
+
+
+IGNORE_DEP_LABELS = ["punct"]
+
+
+
+DEEP_FRAMES = [
+    "Transitive_action",
+    "Causation",
+    "Transition_to_a_state",
+    "Event",
+    "State",
+]
+# SYNTAX_ANALYSIS_CACHE_FILES = {
+#     "femicides/rai": "resources/rai_syntax_analysis_cache.json",
+#     "femicides/rai_main": "resources/rai_main_syntax_analysis_cache.json",
+#     "femicides/olv": "resources/olv_syntax_analysis_cache.json",
+#     "crashes/thecrashes": "resources/thecrashes_syntax_analysis_cache.json",
+#     "migration/pavia": "resources/migration_pavia_syntax_analysis_cache.json"
+# }
+SYNTAX_ANALYSIS_CACHE_FILES = {
+    "femicides/rai": "output/femicides/syntax_cache/rai_ALL",
+    "femicides/rai_main": "output/femicides/syntax_cache/rai_main",
+    "femicides/rai_ALL": "output/femicides/syntax_cache/rai_ALL",
+    "femicides/olv": "output/femicides/syntax_cache/olv",
+    "crashes/thecrashes": "output/crashes/syntax_cache/thecrashes",
+    "migration/pavia": "output/migration/syntax_cache/pavia",
+}
+
+
+DEEP_FRAMES_CACHE_FILE = "resources/deep_frame_cache.json"
+
+DEP_LABEL_CACHE_FILE = "resources/dep_labels.txt"
+
+POSSIBLE_CONSTRUCTIONS = [
+    "nonverbal",
+    "verbal:active",
+    "verbal:impersonal",
+    "verbal:reflexive",
+    "verbal:passive",
+    "verbal:unaccusative",
+    "other",
+]
+
+
+def load_deep_frames_cache():
+    if os.path.isfile(DEEP_FRAMES_CACHE_FILE):
+        print("Loading deep frame cache...")
+        with open(DEEP_FRAMES_CACHE_FILE, encoding="utf-8") as f:
+            deep_frames_cache = json.load(f)
+    else:
+        deep_frames_cache = {}
+    return deep_frames_cache
+
+
+# make spacy work with google app engine
+# (see https://stackoverflow.com/questions/55228492/spacy-on-gae-standard-second-python-exceeds-memory-of-largest-instance)
+# nlp = spacy.load("it_core_news_md")
+nlp = None
+
+
+@dataclasses.dataclass
+class AnnotationSpan:
+    tokens_idx: List[int]
+    tokens_str: List[str]
+
+
+@dataclasses.dataclass
+class FrameStructure:
+    frame: str
+    deep_frame: str
+    target: Optional[AnnotationSpan]
+    roles: List[Tuple[str, AnnotationSpan]]
+    deep_roles: List[Tuple[str, AnnotationSpan]]
+
+
+def make_syntax_cache(dataset, skip_fn=None):
+    print(f"make_syntax_cache({dataset})")
+
+    if dataset == "femicides/rai":
+        corpus_tarball = "output/femicides/lome/lome_0shot/multilabel_rai_blocks"
+        corpus = "rai"
+        spacy_model = "it_core_news_md"
+    elif dataset == "femicides/rai_main":
+        corpus_tarball = "output/femicides/lome/lome_0shot/multilabel_rai_main_blocks"
+        corpus = "rai_main"
+        spacy_model = "it_core_news_md"
+    elif dataset == "femicides/rai_ALL":
+        corpus_tarball = "output/femicides/lome/lome_0shot/multilabel_rai_ALL_blocks"
+        corpus = "rai_ALL"
+        spacy_model = "it_core_news_md"
+    elif dataset == "femicides/olv":
+        corpus_tarball = "output/femicides/lome/lome_0shot/multilabel_olv_blocks"
+        corpus = "olv"
+        spacy_model = "it_core_news_md"
+    elif dataset == "crashes/thecrashes":
+        corpus_tarball = "output/crashes/lome/lome_0shot/multilabel_thecrashes_blocks"
+        corpus = "thecrashes"
+        spacy_model = "nl_core_news_md"
+    elif dataset == "migration/pavia":
+        corpus_tarball = "output/migration/lome/lome_0shot/multilabel_pavia_blocks"
+        # corpus_tarball = "output/migration/lome/lome_zs-tgt_ev-frm/multilabel_pavia.tar.gz"
+        corpus = "pavia"
+        spacy_model = "it_core_news_md"
+    else:
+        raise ValueError("Unsupported dataset!")
+
+    print("params:")
+    print(f"\tcorpus_tarball: {corpus_tarball}")
+    print(f"\tcorpus: {corpus}")
+    print(f"\tspacy: {spacy_model}")
+
+    print("processing files...")
+
+    
+    for block in os.listdir(corpus_tarball):
+        print(block)
+
+        with tarfile.open(os.path.join(corpus_tarball, block)) as tar_in:
+
+            # check if output tarball exists
+            cache_location = SYNTAX_ANALYSIS_CACHE_FILES[dataset]
+            if not os.path.isdir(cache_location):
+                os.makedirs(cache_location)
+
+            lome_files = [f for f in tar_in.getmembers(
+            ) if f.name.endswith(".comm.json")]        
+            
+            lome_files.sort(key=lambda file: file.name)
+            for file in lome_files:
+                print(f"\tprocessing file {file}")
+                doc_id = re.search(r"lome_(\d+)\.comm\.json", file.name).group(1)
+
+                skipped = False
+                if skip_fn is not None:
+                    if skip_fn(doc_id):
+                        print(f"\t\tskip_fn: skipping file {file}")
+                        skipped = True
+
+                if skipped:
+                    syntax_analyses = None
+                else:
+                    file_obj = io.TextIOWrapper(tar_in.extractfile(file))
+                    annotations = json.load(file_obj)
+
+                    syntax_analyses = []
+                    for sentence in annotations:
+                        syntax_analyses.append(
+                            syntax_analyze(sentence, spacy_model))
+
+                # use last two chars of filename as key
+                file_key = doc_id[:2]
+                cache_file = f"{cache_location}/{file_key}.json"
+                if os.path.isfile(cache_file):
+                    with open(cache_file, encoding="utf-8") as f:
+                        key_cache = json.load(f)
+                else:
+                    key_cache = {}
+                key_cache[doc_id] = syntax_analyses
+                with open(cache_file, "w", encoding="utf-8") as f:
+                    json.dump(key_cache, f)
+
+
+def make_syntax_cache_key(filename):
+    doc_id = re.search(r"/\d+/lome_(\d+)\.comm\.json", filename).group(1)
+    return doc_id
+
+
+def clean_sentence_(sentence):
+    idx_to_remove = []
+
+    for i, tok in enumerate(sentence["tokens"]):
+
+        # remove whitespace tokens
+        if not tok.strip():
+            idx_to_remove.append(i)
+
+    idx_to_remove.reverse()
+
+    for idx in idx_to_remove:
+        for annotation_list in sentence.values():
+            annotation_list.pop(idx)
+
+
+def process_prediction_file(
+    filename: str,
+    dataset_name: str,
+    syntax_cache: str,
+    deep_frames_cache: dict,
+    tmp_cache: Optional[dict] = None,
+    file_obj: io.TextIOBase = None,
+    syntax_cache_key: Optional[str] = None,
+    deep_frames_list: Optional[List[str]] = None,
+    spacy_model: str = "it_core_news_md",
+    spacy_model_obj = None
+) -> Tuple[List, ...]:
+    """
+    Process a predictions JSON file
+    :param filename: path to the JSON file
+    :param syntax_cache: see `make_syntax_cache()`
+    :param spacy model: spacy model to be used for syntactic analysis
+    :param file_obj: already opened object corresponding to `filename`. If given, `file_obj` will be used instead
+    of loading it from `filename`. This is useful when reading the entire corpus from a tarball (which is what the
+    SocioFillmore webapp does)
+    :return:
+    """
+
+    print("Processing", filename)
+
+    if file_obj is not None:
+        annotations = json.load(file_obj)
+    else:
+        with open(filename, encoding="utf-8") as f:
+            annotations = json.load(f)
+
+    if syntax_cache is None:
+        syntax_analyses = []
+        for sentence in annotations:
+            syntax_analyses.append(syntax_analyze(sentence, spacy_model, spacy_model_obj))
+
+    else:
+        if syntax_cache_key is None:
+            syntax_cache_key = make_syntax_cache_key(filename)
+
+        if tmp_cache is not None and syntax_cache_key in tmp_cache:
+            syntax_analyses = tmp_cache[syntax_cache_key]
+
+        else:
+            with open(f"{syntax_cache}/{syntax_cache_key[:2]}.json", encoding="utf-8") as cache_file:
+                grouped_analyses = json.load(cache_file)
+                syntax_analyses = grouped_analyses[syntax_cache_key]
+            if tmp_cache is not None:
+                tmp_cache.clear()
+                tmp_cache.update(grouped_analyses)
+
+    fn_structures: List[Dict[int, FrameStructure]] = []
+    sentences: List[List[str]] = []
+    role_analyses: List[Dict[int, Dict[str, str]]] = []
+
+    for sent_idx, sentence in enumerate(annotations):
+
+        clean_sentence_(sentence)
+
+        try:
+            sent_structures = process_fn_sentence(
+                sentence, deep_frames_cache, deep_frames_list=deep_frames_list
+            )
+
+        # seems to occur for one specific file in the migration set, TODO find out what happens
+        except AttributeError:
+            print("Error processing FN annotations")
+            sent_structures = {}
+        syntax = syntax_analyses[sent_idx]
+
+        # disambiguate syntactic constructions
+        for fs in sent_structures.values():
+            target_idx = str(fs.target.tokens_idx[0])
+            if target_idx not in syntax:
+                print(
+                    f"Prediction file {filename}: Cannot find syntactic information for target at idx={target_idx}")
+                continue
+            fs_syn = syntax[target_idx][-1]
+            disambiguate_cxs_(fs, fs_syn)
+
+        roles = process_syn_sem_roles(sent_structures, syntax)
+        role_analyses.append(roles)
+        sentences.append(sentence["tokens"])
+        fn_structures.append(sent_structures)
+
+    return sentences, fn_structures, syntax_analyses, role_analyses
+
+
+def disambiguate_cxs_(struct: FrameStructure, tgt_syntax):
+    # no "_" at the beginning: no disambiguation needed
+    cx = tgt_syntax["syn_construction"]
+    if not cx.startswith("_"):
+        return
+
+    # print(struct.frame, struct.deep_frame)
+
+    # NB works only for the selected relevant frames! if any other frames are added, make sure to update this
+    if struct.deep_frame in ["Transitive_action", "Causation", "Emotion_directed", "Quarreling", "Impact", "Committing_crime"]:
+        frame_agentivity_type = "active"
+    elif struct.frame in ACTIVE_FRAMES:
+        frame_agentivity_type = "active"
+    elif struct.frame == "Event":
+        frame_agentivity_type = "impersonal"
+    else:
+        frame_agentivity_type = "unaccusative"
+
+    if cx == "_verbal:ACTIVE":
+        new_cx = f"verbal:{frame_agentivity_type}"
+    elif cx in ["_verbal:ADPOS", "_verbal:OTH_PART"]:
+        if frame_agentivity_type == "active":
+            new_cx = "verbal:passive"
+        else:
+            new_cx = f"verbal:{frame_agentivity_type}"
+    else:
+        raise ValueError(f"Unknown construction placeholder {cx}")
+
+    tgt_syntax["syn_construction"] = new_cx
+
+
+def find_governed_roles(
+    syn_self: Dict[str, Any],
+    syn_children: List[Dict[str, Any]],
+    roles: List[Tuple[str, AnnotationSpan]],
+) -> Dict[str, str]:
+
+    roles_found = {}
+
+    # find roles that are governed by the predicate
+    for node in [syn_self] + syn_children:
+        for role_name, role_span in roles:
+            if node["lome_idx"] in role_span.tokens_idx:
+                dep_label = node["dependency"]
+                if role_name not in roles_found and dep_label not in IGNORE_DEP_LABELS:
+                    if node == syn_self:
+                        roles_found[role_name] = None
+                    else:
+                        roles_found[role_name] = dep_label + "↓"
+    return roles_found
+
+
+def analyze_role_dependencies(
+    fn_struct,
+    syntax,
+    role_analysis=None,
+    tgt_idx=None,
+    min_depth=-10,
+    max_depth=10,
+    depth=0,
+    label_prefix="",
+):
+
+    if role_analysis is None:
+        role_analysis = {}
+
+    if tgt_idx is None:
+        tgt_idx = fn_struct.target.tokens_idx[0]
+
+    if depth > max_depth:
+        return role_analysis
+
+    if depth < min_depth:
+        return role_analysis
+
+    new_analysis = {}
+    new_analysis.update(role_analysis)
+    token_syntax = syntax[str(tgt_idx)][0]
+
+    def update_analysis(mapping):
+        for role, dep in mapping.items():
+            if role not in new_analysis:
+                if label_prefix:
+                    if dep is None:
+                        label = label_prefix
+                        depth_label = depth
+                    else:
+                        label = label_prefix + "--" + dep
+                        depth_label = depth + 1 if depth > 0 else depth - 1
+                else:
+                    if dep is None:
+                        label = "⋆"
+                        depth_label = depth
+                    else:
+                        label = dep
+                        depth_label = depth + 1 if depth > 0 else depth - 1
+                new_analysis[role] = label, depth_label
+
+    update_analysis(
+        find_governed_roles(
+            token_syntax, token_syntax["children"], fn_struct.roles)
+    )
+
+    # from the initial predicate: first try the children
+    if depth <= 0:
+        for child in token_syntax["children"]:
+            child_analysis = analyze_role_dependencies(
+                fn_struct,
+                syntax,
+                role_analysis=new_analysis,
+                tgt_idx=child["lome_idx"],
+                max_depth=max_depth,
+                min_depth=min_depth,
+                depth=depth - 1,
+                label_prefix=child["dependency"] + "↓"
+            )
+            new_analysis.update(child_analysis)
+
+    # ... then try the ancestors
+    if depth >= 0:
+        if not token_syntax["ancestors"]:
+            return new_analysis
+
+        first_ancestor = token_syntax["ancestors"][0]
+        return analyze_role_dependencies(
+            fn_struct,
+            syntax,
+            role_analysis=new_analysis,
+            tgt_idx=first_ancestor["lome_idx"],
+            max_depth=max_depth,
+            min_depth=min_depth,
+            depth=depth + 1,
+            label_prefix=token_syntax["dependency"] + "↑",
+        )
+
+    else:
+        return new_analysis
+
+
+def process_syn_sem_roles(
+    sent_structures: Dict[int, FrameStructure], syntax: Dict[str, List[Dict[str, Any]]]
+) -> Dict[int, Dict[str, str]]:
+
+    analyses = defaultdict(dict)
+    # go through all frame targets
+    for struct in sent_structures.values():
+        tgt_idx = struct.target.tokens_idx[0]
+        role_deps = analyze_role_dependencies(struct, syntax, max_depth=10)
+        analyses[tgt_idx] = clean_role_deps(role_deps)
+    return analyses
+
+
+def clean_role_deps(role_deps):
+    res = {}
+    for role, (dep_str, depth) in role_deps.items():
+        dep_parts = dep_str.split("--")
+        if len(dep_parts) == 1:
+            res[role] = dep_str, depth
+        else:
+            res[role] = "--".join([dp[-1]
+                                  for dp in dep_parts[:-1]] + [dep_parts[-1]]), depth
+    return res
+
+
+def map_or_lookup_deep_frame(
+    frame: str, deep_frames_cache, save_modified_cache=False, deep_frames_list=None
+) -> Tuple[str, Dict[str, str]]:
+    if frame in deep_frames_cache:
+        return deep_frames_cache[frame]
+    else:
+        deep_frame, mapping = map_to_deep_frame(
+            frame, deep_frames_list=deep_frames_list
+        )
+        deep_frames_cache[frame] = [deep_frame, mapping]
+        if save_modified_cache:
+            with open(DEEP_FRAMES_CACHE_FILE, "w", encoding="utf-8") as f:
+                json.dump(deep_frames_cache, f)
+        return deep_frames_cache[frame]
+
+
+def map_to_deep_frame(
+    frame: str,
+    target: Optional[str] = None,
+    mapping: Optional[Dict[str, str]] = None,
+    self_mapping: Optional[Dict[str, str]] = None,
+    deep_frames_list: Optional[List[str]] = None,
+) -> Tuple[str, Dict[str, str]]:
+
+    if deep_frames_list is None:
+        deep_frames_list = DEEP_FRAMES
+
+    # look up in FrameNet
+    try:
+        fn_entry = fn.frame(frame)
+    except FramenetError:
+        return frame, {}
+    except LookupError:
+        return frame, {}
+
+    # initial call: `target` == `frame`, mapping maps to self
+    if target is None:
+        target = frame
+    if mapping is None or self_mapping is None:
+        mapping = self_mapping = {role: role for role in fn_entry.FE.keys()}
+
+    # base case: our frame is a deep frame
+    if frame in deep_frames_list:
+        return frame, mapping
+
+    # otherwise, look at parents
+    inh_relations = [
+        fr
+        for fr in fn_entry.frameRelations
+        if fr.type.name == "Inheritance" and fr.Child == fn_entry
+    ]
+    parents = [fr.Parent for fr in inh_relations]
+
+    # no parents --> failure, return original frame
+    if not inh_relations:
+        return target, self_mapping
+
+    # one parent: follow that parent
+    if len(inh_relations) == 1:
+        parent_rel = inh_relations[0]
+        parent = parents[0]
+        new_mapping = define_fe_mapping(mapping, parent_rel)
+        return map_to_deep_frame(
+            parent.name, target, new_mapping, self_mapping, deep_frames_list
+        )
+
+    # more parents: check if any of them leads to a deep frame
+    deep_frames = []
+    deep_mappings = []
+    for parent_rel, parent in zip(inh_relations, parents):
+        new_mapping = define_fe_mapping(mapping, parent_rel)
+        final_frame, final_mapping = map_to_deep_frame(
+            parent.name, target, new_mapping, self_mapping, deep_frames_list
+        )
+        if final_frame in deep_frames_list:
+            deep_frames.append(final_frame)
+            deep_mappings.append(final_mapping)
+
+    for deep_frame in deep_frames_list:
+        if deep_frame in deep_frames:
+            idx = deep_frames.index(deep_frame)
+            return deep_frame, deep_mappings[idx]
+
+    # nothing found, return original frame
+    return target, self_mapping
+
+
+def define_fe_mapping(mapping, parent_rel):
+    child_to_parent_mapping = {
+        fer.subFEName: fer.superFEName for fer in parent_rel.feRelations
+    }
+    target_to_parent_mapping = {
+        role: child_to_parent_mapping[mapping[role]]
+        for role in mapping
+        if mapping[role] in child_to_parent_mapping
+    }
+    return target_to_parent_mapping
+
+
+def is_at_root(syntax_info):
+
+    # you should either be the actual root...
+    if syntax_info["dependency"] == "ROOT":
+        return True
+
+    # ... or be the subject of the root
+    if syntax_info["dependency"] == "nsubj" and syntax_info["ancestors"][0]["dependency"] == "ROOT":
+        return True
+
+    return False
+
+
+def get_tarball_blocks(dataset, lome_model="lome_0shot"):
+    if dataset == "femicides/rai":
+        return f"output/femicides/lome/{lome_model}/multilabel_rai_ALL_blocks"
+    if dataset == "femicides/rai_main":
+        return f"output/femicides/lome/{lome_model}/multilabel_rai_main_blocks"
+    elif dataset == "femicides/olv":
+        return f"output/femicides/lome/{lome_model}/multilabel_olv_blocks"
+    elif dataset == "crashes/thecrashes":
+        return f"output/crashes/lome/{lome_model}/multilabel_thecrashes_blocks"
+    elif dataset == "migration/pavia":
+        return f"output/migration/lome/{lome_model}/multilabel_pavia_blocks"
+    else:
+        raise ValueError("Unsupported dataset!")
+
+
+def analyze_single_document(doc_id, event_id, lome_model, dataset, texts_df, deep_frames_cache):
+    data_domain, data_corpus = dataset.split("/")
+
+    syntax_cache = SYNTAX_ANALYSIS_CACHE_FILES[dataset]
+
+    print(dataset)
+
+    if dataset == "migration/pavia":  # this is a hack, fix it!
+        pred_file_path = f"output/migration/lome/multilabel/{lome_model}/pavia/{event_id}/lome_{doc_id}.comm.json"
+    elif dataset == "femicides/olv":
+        pred_file_path = f"output/femicides/lome/lome_0shot/multilabel/olv/{event_id}/lome_{doc_id}.comm.json"
+    else:
+        pred_file_path = f"output/{data_domain}/lome/lome_0shot/multilabel/{data_corpus}/{event_id}/lome_{doc_id}.comm.json"
+    print(f"Analyzing file {pred_file_path}")
+
+    doc_id = os.path.basename(pred_file_path).split(".")[0].split("_")[1]
+    doc_key = doc_id[:2]
+    tarball = get_tarball_blocks(dataset, lome_model) + f"/block_{doc_key}.tar"
+    with tarfile.open(tarball, "r") as tar_f:
+        pred_file = io.TextIOWrapper(tar_f.extractfile(pred_file_path))
+
+        (
+            sents,
+            pred_structures,
+            syntax_analyses,
+            role_analyses,
+        ) = process_prediction_file(
+            filename=pred_file_path,
+            dataset_name=dataset,
+            file_obj=pred_file,
+            syntax_cache=syntax_cache,
+            deep_frames_cache=deep_frames_cache
+        )
+        output = []
+        for sent, structs, syntax, roles in zip(
+            sents, pred_structures, syntax_analyses, role_analyses
+        ):
+            output.append(
+                {
+                    "sentence": sent,
+                    "fn_structures": [
+                        dataclasses.asdict(fs) for fs in structs.values()
+                    ],
+                    "syntax": syntax,
+                    "roles": roles,
+                    "meta": {
+                        "event_id": event_id,
+                        "doc_id": doc_id,
+                        "text_meta": get_text_meta(doc_id, texts_df),
+                    },
+                }
+            )
+    return output
+
+
+def get_text_meta(doc_id, texts_df):
+    row = texts_df[texts_df["text_id"] == int(doc_id)].iloc[0]
+    if "pubdate" in row:
+        pubdate = row["pubdate"] if not pd.isna(row["pubdate"]) else None
+    elif "pubyear" in row:
+        pubdate = int(row["pubyear"])
+    else:
+        pubdate = None
+    return {
+        "url": row["url"] if "url" in row else None,
+        "pubdate": pubdate,
+        "provider": row["provider"],
+        "title": row["title"] if not pd.isna(row["title"]) else None,
+        "days_after_event": int(row["days_after_event"]) if "days_after_event" in row and not pd.isna(row["days_after_event"]) else 0
+    }
+
+
+def process_fn_sentence(
+    sentence, deep_frames_cache, post_process=True, deep_frames_list=None
+):
+    # frame structures in the sentence
+    sent_structures: Dict[int, FrameStructure] = {}
+
+    # role spans currently being built up (per structure + role name)
+    cur_spans: Dict[Tuple[int, str]] = {}
+    for token_idx, (token_str, frame_annos) in enumerate(
+        zip(sentence["tokens"], sentence["frame_list"])
+    ):
+        for fa in frame_annos:
+            # remove "virtual root" nonsense token
+            if "@@VIRTUAL_ROOT@@" in fa:
+                continue
+            fa = fa.split("@@")[0]  # remove confidence score if it's there
+            anno, struct_id_str = fa.split("@")
+            struct_id = int(struct_id_str)
+            frame_name = anno.split(":")[1]
+            deep_frame, deep_frame_mapping = map_or_lookup_deep_frame(
+                frame_name, deep_frames_cache, deep_frames_list=deep_frames_list
+            )
+            if struct_id not in sent_structures:
+                sent_structures[struct_id] = FrameStructure(
+                    frame=frame_name,
+                    deep_frame=deep_frame,
+                    target=None,
+                    roles=[],
+                    deep_roles=[],
+                )
+            cur_struct = sent_structures[struct_id]
+
+            # TODO: get rid of this hack
+            anno = anno.replace("I::", "I:")
+            anno = anno.replace("B::", "B:")
+
+            if anno.split(":")[0] == "T":
+                if cur_struct.target is None:
+                    cur_struct.target = AnnotationSpan(
+                        [token_idx], [token_str])
+                else:
+                    cur_struct.target.tokens_idx.append(token_idx)
+                    cur_struct.target.tokens_str.append(token_str)
+            elif anno.split(":")[0] == "B":
+                role_name = anno.split(":")[2]
+                role_span = AnnotationSpan([token_idx], [token_str])
+                cur_struct.roles.append((role_name, role_span))
+                if role_name in deep_frame_mapping:
+                    cur_struct.deep_roles.append(
+                        (deep_frame_mapping[role_name], role_span)
+                    )
+                cur_spans[(struct_id, role_name)] = role_span
+            elif anno.split(":")[0] == "I":
+                role_name = anno.split(":")[2]
+                role_span = cur_spans[(struct_id, role_name)]
+                role_span.tokens_str.append(token_str)
+                role_span.tokens_idx.append(token_idx)
+
+    # post-process: remove punctuation in targets
+    if post_process:
+        for fs in sent_structures.values():
+            if len(fs.target.tokens_str) > 1:
+                target_tok_str_to_remove = []
+                target_tok_idx_to_remove = []
+                for tok_str, tok_idx in zip(fs.target.tokens_str, fs.target.tokens_idx):
+                    if tok_str in ["``", "''", "`", "'", ".", ",", ";", ":"]:
+                        target_tok_str_to_remove.append(tok_str)
+                        target_tok_idx_to_remove.append(tok_idx)
+                for tok_str, tok_idx in zip(
+                    target_tok_str_to_remove, target_tok_idx_to_remove
+                ):
+                    fs.target.tokens_str.remove(tok_str)
+                    fs.target.tokens_idx.remove(tok_idx)
+
+    return sent_structures
+
+
+def map_back_spacy_lome_tokens(spacy_doc, lome_tokens):
+    if len(lome_tokens) > len(spacy_doc):
+        raise ValueError(
+            f"Cannot re-tokenize (#lome={len(lome_tokens)} // #spacy={len(spacy_doc)})"
+        )
+
+    spacy_to_lome = {}
+    lome_idx = 0
+    for spacy_idx, spacy_token in enumerate(spacy_doc):
+        spacy_to_lome[spacy_idx] = lome_idx
+
+        # whitespace after token: tokens correspond
+        if spacy_token.whitespace_:
+            lome_idx += 1
+    return spacy_to_lome
+
+
+def get_syn_category(spacy_token):
+    if spacy_token.pos_ == "NOUN":
+        return "n"
+    if spacy_token.pos_ == "ADJ":
+        return "adj"
+    if spacy_token.pos_ == "ADV":
+        return "adv"
+    if spacy_token.pos_ == "ADP":
+        return "p"
+    if spacy_token.pos_ == "VERB":
+        if spacy_token.morph.get("VerbForm") == ["Fin"]:
+            return "v:fin"
+        if spacy_token.morph.get("VerbForm") == ["Part"]:
+            return "v:part"
+        if spacy_token.morph.get("VerbForm") == ["Ger"]:
+            return "v:ger"
+        if spacy_token.morph.get("VerbForm") == ["Inf"]:
+            return "v:inf"
+    return "other"
+
+
+def syntax_analyze(sentence, spacy_model_name, spacy_model_obj=None) -> Dict[str, Dict[str, Any]]:
+    lome_tokens = sentence["tokens"]
+
+    # load spacy model locally (so that it works in GAE)
+    # global nlp
+    if spacy_model_obj is not None:
+        nlp = spacy_model_obj
+    else:
+        nlp = spacy.load(spacy_model_name)
+
+    spacy_doc = nlp(" ".join(lome_tokens))
+    analysis = defaultdict(list)
+    spacy_to_lome_tokens = map_back_spacy_lome_tokens(spacy_doc, lome_tokens)
+    for spacy_idx, token in enumerate(spacy_doc):
+        lome_idx = spacy_to_lome_tokens[spacy_idx]
+        syn_category = get_syn_category(token)
+        syn_construction = get_syn_construction(token, syn_category)
+        children = []
+        for c in token.children:
+            children.append(
+                {
+                    "token": c.text,
+                    "spacy_idx": c.i,
+                    "lome_idx": spacy_to_lome_tokens[c.i],
+                    "syn_category": get_syn_category(c),
+                    "dependency": c.dep_,
+                }
+            )
+        ancestors = []
+        for a in token.ancestors:
+            ancestors.append(
+                {
+                    "token": a.text,
+                    "spacy_idx": a.i,
+                    "lome_idx": spacy_to_lome_tokens[a.i],
+                    "syn_category": get_syn_category(a),
+                    "dependency": a.dep_,
+                }
+            )
+
+        # str key so that it doesn't change when converting to JSON
+        lome_key = str(lome_idx)
+        analysis[lome_key].append(
+            {
+                "token": token.text,
+                "dependency": token.dep_,
+                "spacy_idx": spacy_idx,
+                "lome_idx": lome_idx,
+                "syn_category": syn_category,
+                "syn_construction": syn_construction,
+                "children": children,
+                "ancestors": ancestors,
+            }
+        )
+    return analysis
+
+
+def get_syn_construction(token: Token, syn_category: str) -> str:
+    if syn_category in ["n", "adj", "adv", "p"]:
+        return "nonverbal"
+
+    if syn_category.startswith("v:"):
+        # find reflexives
+        for c in token.children:
+            if c.lemma_.lower() in ["si", "zich", "zichzelf"]:
+                return "verbal:reflexive"
+
+        # find impersonal constructions
+        for c in token.children:
+            if c.dep_ == "expl":
+                return "verbal:impersonal"
+
+    # all other finite verbs/gerunds/infinites -> active construction
+    if syn_category in ["v:fin", "v:ger", "v:inf"]:
+        return "_verbal:ACTIVE"
+
+    if syn_category == "v:part":
+
+        if token.dep_ == "acl":
+            return "_verbal:ADPOS"
+
+        for c in token.children:
+
+            # passive subj or auxiliary present: it's a passive
+            if c.dep_ in ["nsubj:pass", "aux:pass"]:
+                return "verbal:passive"
+
+            # auxiliary "HAVE" (avere/hebben) present: it's an active
+            if (
+                c.dep_ == "aux"
+                and c.lemma_.lower() in ITALIAN_ACTIVE_AUX + DUTCH_ACTIVE_AUX
+            ):
+                return "verbal:active"
+
+        return "_verbal:OTH_PART"
+
+    return "other"
+
+
+def get_syntax_info(struct: FrameStructure, syntax: Dict) -> Dict:
+    target_idx = str(struct.target.tokens_idx[0])
+    # print(target_idx, syntax)
+    syntax_for_target = syntax[target_idx]
+    return syntax_for_target[-1]
+
+
+def enrich_texts_df(texts_df: pd.DataFrame, events_df: pd.DataFrame):
+    time_delta_rows: List[Optional[int]] = []
+    for idx, text_row in texts_df.iterrows():
+        try:
+            event_row = events_df[events_df["event:id"]
+                              == text_row["event_id"]].iloc[0]
+        except IndexError:
+            print(f"Skipping {idx} (IndexError)")
+            time_delta_rows.append(None)
+        if "pubdate" not in text_row or pd.isna(text_row["pubdate"]) or pd.isna(event_row["event:date"]):
+            time_delta_rows.append(None)
+        else:
+            try:
+                pub_date = datetime.strptime(
+                    text_row["pubdate"], "%Y-%m-%d %H:%M:%S")
+                event_date = datetime.strptime(
+                    event_row["event:date"], "%Y-%m-%d")
+                time_delta = pub_date - event_date
+                time_delta_days = time_delta.days
+                time_delta_rows.append(time_delta_days)
+            except ValueError as e:
+                print(
+                    f"\t\terror parsing dates, see below for more info:\n\t\t{e}")
+                time_delta_rows.append(None)
+
+    return texts_df.assign(days_after_event=time_delta_rows)
+
+
+def read_frames_of_interest(dataset) -> List[str]:
+    if dataset in ["femicides/rai", "femicides/olv"]:
+        file = "resources/femicide_frame_list.txt"
+    elif dataset == "crashes/thecrashes":
+        file = "resources/crashes_frame_list.txt"
+    elif dataset == "migration/pavia":
+        file = "resources/migration_frame_list.txt"
+    else:
+        raise ValueError("Unsupported dataset")
+
+    frames = set()
+    with open(file, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line.startswith("#") or not line:
+                continue
+            frames.add(line[0].upper() + line[1:].lower())
+    return sorted(frames)
+
+
+def make_dep_label_cache():
+
+    labels = set()
+
+    for dataset in ["femicides/rai", "crashes/thecrashes", "migration/pavia"]:
+
+        tarball = (
+            "output/femicides/lome/lome_0shot/multilabel_rai.tar.gz"
+            if dataset == "femicides/rai"
+            else "output/crashes/lome/lome_0shot/multilabel_thecrashes.tar.gz"
+            if dataset == "crashes/thecrashes"
+            else "output/migration/lome/lome_0shot/multilabel_pavia.tar.gz"
+        )
+
+        spacy_model = (
+            "it_core_news_md" if dataset["femicides/rai",
+                                         "migration/pavia"] else "nl_core_news_md"
+        )
+
+        deep_frames_cache = load_deep_frames_cache(dataset)
+        syntax_cache = SYNTAX_ANALYSIS_CACHE_FILES[dataset]
+
+        with tarfile.open(tarball, "r:gz") as tar_f:
+            for mem in [
+                m.name for m in tar_f.getmembers() if m.name.endswith(".comm.json")
+            ]:
+                if mem is None:
+                    continue
+
+                print(mem)
+                mem_obj = io.TextIOWrapper(tar_f.extractfile(mem))
+                (_, _, _, role_analyses,) = process_prediction_file(
+                    filename=mem,
+                    dataset_name=dataset,
+                    file_obj=mem_obj,
+                    syntax_cache=syntax_cache,
+                    deep_frames_cache=deep_frames_cache,
+                    spacy_model=spacy_model,
+                )
+                if role_analyses is None:
+                    print(f"\tSkipping file {mem}, no role analyses found")
+                    continue
+                for sent_ra in role_analyses:
+                    for ra in sent_ra.values():
+                        for dep, _ in ra.values():
+                            labels.add(dep)
+    with open(DEP_LABEL_CACHE_FILE, "w", encoding="utf-8") as f_out:
+        for label in sorted(labels):
+            f_out.write(label + os.linesep)
+
+
+def analyze_external_file(file_in, file_out, spacy_model):
+    deep_frames_cache = load_deep_frames_cache()
+    (
+        sents,
+        pred_structures,
+        syntax_analyses,
+        role_analyses,
+    ) = process_prediction_file(file_in, "", None, deep_frames_cache, spacy_model_obj=spacy_model)
+    output = []
+    for sent, structs, syntax, roles in zip(
+        sents, pred_structures, syntax_analyses, role_analyses
+    ):
+        output.append(
+            {
+                "sentence": sent,
+                "fn_structures": [
+                    dataclasses.asdict(fs) for fs in structs.values()
+                ],
+                "syntax": syntax,
+                "roles": roles
+            }
+        )
+    with open(file_out, "w", encoding="utf-8") as f_out:
+        json.dump(output, f_out, indent=4)
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("command", choices=[
+                    "make_syntax_cache", "make_dep_label_cache", "analyze_file"
+                    ])
+    ap.add_argument("dataset", choices=["femicides/rai", "femicides/rai_main", "femicides/rai_ALL",
+                    "femicides/olv", "crashes/thecrashes", "migration/pavia", "*"])
+    ap.add_argument("--input_file", type=str, default="")
+    ap.add_argument("--output_file", type=str, default="")
+    args = ap.parse_args()
+
+    if args.command == "make_syntax_cache":
+
+        if args.dataset == "*":
+            raise ValueError(
+                "Please specificy a dataset for `make_syntax_cache`")
+
+        if args.dataset == "crashes/thecrashes":
+            make_syntax_cache(
+                "crashes/thecrashes", skip_fn=lambda f: not is_a_dutch_text(f)
+            )
+        elif args.dataset == "femicides/rai":
+            make_syntax_cache("femicides/rai")
+        elif args.dataset == "femicides/rai_main":
+            make_syntax_cache("femicides/rai_main")
+        elif args.dataset == "femicides/rai_ALL":
+            make_syntax_cache("femicides/rai_ALL")
+        elif args.dataset == "femicides/olv":
+            make_syntax_cache("femicides/olv")
+        else:
+            make_syntax_cache("migration/pavia")
+
+    elif args.command == "make_dep_label_cache":
+        make_dep_label_cache()
+
+    elif args.command == "analyze_file":
+        analyze_external_file(args.input_file, args.output_file)
+        
+
+

sociofillmore/common/convert_comms.py

@@ -0,0 +1,208 @@
+"""
+Adapted from comm2multilabel.py from the Bert-for-FrameNet project (https://gitlab.com/gosseminnema/bert-for-framenet)
+"""
+
+import dataclasses
+import json
+import os
+import glob
+import sys
+from collections import defaultdict
+from typing import List, Optional
+
+import nltk
+from concrete import Communication
+from concrete.util import read_communication_from_file, lun, get_tokens
+
+
+@dataclasses.dataclass
+class FrameAnnotation:
+    tokens: List[str] = dataclasses.field(default_factory=list)
+    pos: List[str] = dataclasses.field(default_factory=list)
+
+
+@dataclasses.dataclass
+class MultiLabelAnnotation(FrameAnnotation):
+    frame_list: List[List[str]] = dataclasses.field(default_factory=list)
+    lu_list: List[Optional[str]] = dataclasses.field(default_factory=list)
+
+    def to_txt(self):
+        for i, tok in enumerate(self.tokens):
+            yield f"{tok} {self.pos[i]} {'|'.join(self.frame_list[i]) or '_'} {self.lu_list[i] or '_'}"
+
+    @staticmethod
+    def from_txt(sentence_lines):
+
+        tokens = []
+        pos = []
+        frame_list = []
+        lu_list = []
+        for line in sentence_lines:
+
+            # ignore any spaces
+            if line.startswith(" "):
+                continue
+
+            columns = line.split()
+            tokens.append(columns[0])
+            pos.append(columns[1])
+
+            # read frame list, handle empty lists
+            if columns[2] == "_":
+                frame_list.append([])
+            else:
+                frame_list.append(columns[2].split("|"))
+
+            # read lu list, handle nulls
+            if columns[3] == "_":
+                lu_list.append(None)
+            else:
+                lu_list.append(columns[3])
+        return MultiLabelAnnotation(tokens, pos, frame_list, lu_list)
+
+    def get_label_set(self):
+        label_set = set()
+        for tok_labels in self.frame_list:
+            for label in tok_labels:
+                label_set.add(label)
+        return label_set
+
+
+def convert_file(file, language="english", confidence_filter=0.0):
+    print("Reading input file...")
+    comm = read_communication_from_file(file)
+
+    print("Mapping sentences to situations...")
+    tok_uuid_to_situation = map_sent_to_situation(comm)
+
+    print("# sentences with situations:", len(tok_uuid_to_situation))
+
+    for section in lun(comm.sectionList):
+        for sentence in lun(section.sentenceList):
+            tokens = get_tokens(sentence.tokenization)
+            situations = tok_uuid_to_situation[sentence.tokenization.uuid.uuidString]
+            tok_to_annos = map_tokens_to_annotations(comm, situations, confidence_filter)
+
+            frame_list, tok_list = prepare_ml_lists(language, tok_to_annos, tokens)
+
+            ml_anno = MultiLabelAnnotation(tok_list, ["_" for _ in tok_list], frame_list,
+                                           [None for _ in tok_list])
+            yield ml_anno
+
+
+def prepare_ml_lists(language, tok_to_annos, tokens):
+    tok_list = []
+    frame_list = []
+    for tok_idx, tok in enumerate(tokens):
+        # split tokens that include punctuation
+        split_tok = nltk.word_tokenize(tok.text, language=language)
+        tok_list.extend(split_tok)
+        tok_anno = []
+        for anno in tok_to_annos.get(tok_idx, []):
+            tok_anno.append(anno)
+        frame_list.extend([list(tok_anno) for _ in split_tok])
+
+    # remove annotations from final punctuation & solve BIO weird stuff
+    for idx, (tok, frame_annos) in enumerate(zip(tok_list, frame_list)):
+        if tok in ",.:;\"'`«»":
+            to_delete = []
+            for fa in frame_annos:
+                if fa.startswith("T:"):
+                    compare_fa = fa
+                else:
+                    compare_fa = "I" + fa[1:]
+
+                if idx == len(tok_list) - 1:
+                    to_delete.append(fa)
+                elif compare_fa not in frame_list[idx + 1]:
+                    to_delete.append(fa)
+
+            for fa in to_delete:
+                frame_annos.remove(fa)
+
+        for fa_idx, fa in enumerate(frame_annos):
+
+            if fa.startswith("B:"):
+                # check if we had exactly the same label the token before
+                if idx > 0 and fa in frame_list[idx - 1]:
+                    frame_annos[fa_idx] = "I" + fa[1:]
+
+    return frame_list, tok_list
+
+
+def map_tokens_to_annotations(comm: Communication, situations: List[str], confidence_filter: float):
+    tok_to_annos = defaultdict(list)
+    for sit_idx, sit_uuid in enumerate(situations):
+        situation = comm.situationMentionForUUID[sit_uuid]
+        if situation.confidence < confidence_filter:
+            continue
+
+        frame_type = situation.situationKind
+        tgt_tokens = situation.tokens.tokenIndexList
+
+        if frame_type == "@@VIRTUAL_ROOT@@":
+            continue
+
+        for tok_id in tgt_tokens:
+            tok_to_annos[tok_id].append(f"T:{frame_type}@{sit_idx:02}@@{situation.confidence}")
+        for arg in situation.argumentList:
+            if arg.confidence < confidence_filter:
+                continue
+
+            fe_type = arg.role
+            fe_tokens = arg.entityMention.tokens.tokenIndexList
+            for tok_n, tok_id in enumerate(fe_tokens):
+                if tok_n == 0:
+                    bio = "B"
+                else:
+                    bio = "I"
+                tok_to_annos[tok_id].append(f"{bio}:{frame_type}:{fe_type}@{sit_idx:02}@@{arg.confidence}")
+    return tok_to_annos
+
+
+def map_sent_to_situation(comm):
+    tok_uuid_to_situation = defaultdict(list)
+    for situation in comm.situationMentionSetList:
+        for mention in situation.mentionList:
+            tok_uuid_to_situation[mention.tokens.tokenizationId.uuidString].append(mention.uuid.uuidString)
+    return tok_uuid_to_situation
+
+
+def main():
+    file_in = sys.argv[1]
+    language = sys.argv[2]
+    output_directory = sys.argv[3]
+    confidence_filter = float(sys.argv[4])
+    split_by_migration_files = False
+
+    file_in_base = os.path.basename(file_in)
+    file_out = f"{output_directory}/lome_{file_in_base}"
+    multi_label_annos = list(convert_file(file_in, language=language, confidence_filter=confidence_filter))
+    multi_label_json = [dataclasses.asdict(anno) for anno in multi_label_annos]
+
+    if split_by_migration_files:
+        files = glob.glob("output/migration/split_data/split_dev10_sep_txt_files/*.orig.txt")
+        files.sort(key=lambda f: int(f.split("/")[-1].rstrip(".orig.txt")))
+
+        for anno, file in zip(multi_label_annos, files):
+            basename = file.split("/")[-1].rstrip(".orig.txt")
+            spl_file_out = f"{output_directory}/{basename}"
+            with open(f"{spl_file_out}.txt", "w", encoding="utf-8") as f_txt:
+                for line in anno.to_txt():
+                    f_txt.write(line + os.linesep)
+                f_txt.write(os.linesep)
+
+    else:
+        print(file_out)
+        with open(f"{file_out}.json", "w", encoding="utf-8") as f_json:
+            json.dump(multi_label_json, f_json, indent=4)
+
+        with open(f"{file_out}.txt", "w", encoding="utf-8") as f_txt:
+            for anno in multi_label_annos:
+                for line in anno.to_txt():
+                    f_txt.write(line + os.linesep)
+                f_txt.write(os.linesep)
+
+
+if __name__ == '__main__':
+    main()

sociofillmore/common/filter_lang.py

@@ -0,0 +1,32 @@
+import langdetect
+
+import json
+
+
+DATA_FILE = "data/thecrashes_data.json"
+
+
+def main():
+    texts = get_texts()
+    for text in texts:
+        if langdetect.detect(text) == "en":
+            print("\n<-------------------------------")
+            print(text)
+            print("------------------------------>\n")
+
+
+def get_texts():
+    with open(DATA_FILE, encoding="utf-8") as f:
+        data = json.load(f)
+
+    texts = []
+
+    for event in data:
+        for article in event["articles"]:
+            texts.append(article["title"] + "\n\n" + article["summary"])
+
+    return texts
+
+
+if __name__ == '__main__':
+    main()

sociofillmore/common/get_nltk_fn_roles.py

@@ -0,0 +1,11 @@
+from nltk.corpus import framenet as fn
+import json
+
+frames_to_roles = {}
+
+for frame in fn.frames():
+    frames_to_roles[frame.name] = list(frame.FE.keys())
+
+
+with open("resources/fn_frames_to_roles.json", "w", encoding="utf-8") as f:
+    json.dump(frames_to_roles, f)

sociofillmore/common/pos_based_targetid.py

@@ -0,0 +1,31 @@
+import os
+import json
+
+import spacy
+
+
+TARGET_POS = [
+    "NOUN",
+    "VERB",
+    "ADJ",
+    "ADV"
+]
+
+
+def do_frameid():
+    nlp = spacy.load("it_core_news_md")
+
+    with open("data/migration/corpus_titoli_all_raw.txt", encoding="utf-8") as f_in, \
+        open("output/migration/pos_based_targetid/corpus_titoli_all_raw.jsonl", "w", encoding="utf-8") as f_out:
+        
+        for line in f_in:
+            doc = nlp(line.strip())
+            out = {
+                "tokens": [t.text for t in doc],
+                "predicates": [i for i, t in enumerate(doc) if t.pos_ in TARGET_POS]  
+            }
+            f_out.write(json.dumps(out) + os.linesep)
+
+
+if __name__ == "__main__":
+    do_frameid()

sociofillmore/common/split_lome_files.py

@@ -0,0 +1,22 @@
+import os
+import glob
+import tarfile
+
+
+def split_lome_files(lome_folder, output_folder):
+    for file in glob.glob(f"{lome_folder}/**/*.comm.*"):
+        doc_id = os.path.basename(file).split(".")[0].split("_")[1]
+        doc_key = doc_id[:2]
+
+        print(file, "->", doc_key)
+
+        with tarfile.open(f"{output_folder}/block_{doc_key}.tar", "a") as tar_f:
+            tar_f.add(file)
+
+
+if __name__ == "__main__":
+    #split_lome_files("output/migration/lome/multilabel/lome_0shot/pavia/", "output/migration/lome/lome_0shot/multilabel_pavia_blocks")
+    # split_lome_files("output/femicides/lome/lome_0shot/multilabel/rai/", "output/femicides/lome/lome_0shot/multilabel_rai_blocks")
+    split_lome_files("output/femicides/lome/lome_0shot/multilabel/rai_ALL/", "output/femicides/lome/lome_0shot/multilabel_rai_ALL_blocks")
+    # split_lome_files("output/femicides/lome/lome_0shot/multilabel/olv/", "output/femicides/lome/lome_0shot/multilabel_olv_blocks")
+    # split_lome_files("output/crashes/lome/lome_0shot/multilabel/thecrashes/", "output/crashes/lome/lome_0shot/multilabel_thecrashes_blocks")

sociofillmore/crashes/generate_templates.py

@@ -0,0 +1,277 @@
+import random
+import json
+import os
+
+random.seed(2021)
+
+
+NUM_SENTENCES = 100_000
+NUM_FAILS = 25
+
+
+SENT_TYPES = ("0_PTY", "1_PTY", "2_PTY")
+SENT_1_PTY_TYPES = ("VICTIM", "OUTCOME", "DRIVE")
+SENT_ACTIVE_TYPES = ("ACTIVE", "NON_ACTIVE")
+
+SENTS_0_PTY_OUTCOME = ("[[OUTCOME]] [[CIRCUMSTANCE]] [[PLACE]]",
+                       "[[OUTCOME]] [[CIRCUMSTANCE]] [[TIME]]", "[[OUTCOME]] [[CIRCUMSTANCE]]")
+SENTS_1_PTY_VICTIM = ("[[SUBJECT]] [[VERB_V2]] [[PLACE]]",
+                      "[[SUBJECT]] [[TIME]] [[VERB_V2]]", "[[SUBJECT]] [[VERB_V2]]")
+SENTS_1_PTY_OUTCOME = ("[[SUBJECT]] [[OUTCOME]] [[PLACE]] [[CIRCUMSTANCE]]",
+                       "[[SUBJECT]] [[OUTCOME]] [[CIRCUMSTANCE]]")
+SENTS_1_PTY_DRIVE = ("[[SUBJECT]] [[VP_DRIVE]] [[PLACE]]",
+                     "[[SUBJECT]] [[VP_DRIVE]]")
+SENTS_2_PTYS = ("[[SUBJECT]] [[VERB_V2]] [[VERB_P]] [[OTHER]] [[VERB_REST]] [[PLACE]]",
+                "[[SUBJECT]] [[VERB_V2]] [[TIME]] [[VERB_P]] [[OTHER]] [[VERB_REST]]", "[[SUBJECT]] [[VERB_V2]] [[VERB_P]] [[OTHER]] [[VERB_REST]]")
+
+PLACES = ("op stationsplein", "in stadscentrum", "op kruispunt Westerhaven", "op A27", "op A10", "in Lelystad",
+          "in Assen", "in Amsterdam", "bij Renkum", "in Schilderswijk", "bij knooppunt Lunetten", "op zuidelijke ringweg",
+          "in de buurt van de Erasmusbrug", "op schoolplein Stedelijk Gymnasium", "bij afrit Rotterdam-Noord", "op Kanaleneiland")
+TIMES = ("tijdens avondspits", "vrijdagavond",
+         "dinsdagochtend", "donderdagnacht", "rond middaguur")
+CIRCUMSTANCES = ("na ongeluk", "na aanrijding", "na botsing", "na crash")
+CIRCUMSTANCES_AGT = (", dader ervandoor", ", dader ervandoor", ", dader rijdt door", ", bestuurder rijdt door")
+
+OUTCOME_0_TYPES = ("TRAFFIC", "HUMAN")
+OUTCOMES_0_TRAFFIC = ("verkeersopstopping", "file", "veel vertraging")
+OUTCOMES_0_HUMAN = ("dode", "zwaargewonde", "gewonde", "drie gewonden")
+OUTCOMES_1 = ("dood", "overleden", "zwaargewond", "lichtgewond", "ongedeerd")
+
+SUBJECT_TYPES = ("WEAK_PTY", "DRIVER", "VERHICLE")
+
+VPS_DRIVE_ACTIVE = ("rijdt tegen boom", "veroorzaakt ongeluk")
+VPS_DRIVE_NON_ACTIVE = ("verongelukt", "gecrasht", "uit de bocht gevlogen", "raakt gewond", "raakt gewond door klap")
+EVENT_VERBS_1_VICTIM = ("aangereden", "geschept", "raakt gewond", "raakt gewond door klap")
+EVENT_VERBS_2_ACTIVE_ANY = ("raakt|_|_", "botst|op|_", "botst|tegen|_")
+EVENT_VERBS_2_ACTIVE_DRIVE = ("rijdt|_|aan", "rijdt|_|dood", "schept|_|_")
+EVENT_VERBS_2_NON_ACTIVE_DRIVER = (
+    "aangereden|door|_", "geschept|door|_")
+EVENT_VERBS_2_NON_ACTIVE_VEHICLE = (
+    "aangereden|door|_", "geschept|door|_", "komt|onder|_")
+EVENT_VERBS_2_NON_ACTIVE_ANY = (
+    "geraakt|door|_",)
+
+
+
+WEAK_PTY_NPS = ("fietser", "skateboarder", "wielrenner", "rolschaatser", "jogger", "voetganger", "motorrijder",
+                "fietskoerier", "[[PERSON]] op fiets", "[[PERSON]] op e-bike")
+ANY_PERSON_NPS = ("vrouw", "man", "meisje", "jongen",
+                  "bejaarde vrouw", "bejaarde man", "Duitser", "toerist")
+CYCLIST_PERSON_NPS = ("postbode", "maaltijdbezorger", "politieagent")
+DRIVER_NPS = ("automobilist", "automobiliste", "bestuurder", "dronken automobilist", "dronken bestuurder", "motorrijder",
+              "minderjarige bestuurder", "trucker", "taxichauffeur", "[[PERSON]] in auto", "dronken [[PERSON]] in auto")
+VEHICLE_NPS = ("auto", "personenauto", "vrachtwagen", "tractor", "auto met caravan", "scooter", "motor",
+               "tram", "stadsbus", "lijn 10", "touringcar", "camper", "vorkheftruck")
+
+
+def generate_weak_pty():
+    noun_phrase = random.choice(WEAK_PTY_NPS)
+    if "[[PERSON]]" in noun_phrase:
+        person = random.choice(ANY_PERSON_NPS + CYCLIST_PERSON_NPS)
+        return noun_phrase.replace("[[PERSON]]", person)
+    else:
+        return noun_phrase
+
+
+def generate_driver():
+    noun_phrase = random.choice(DRIVER_NPS)
+    if "[[PERSON]]" in noun_phrase:
+        person = random.choice(ANY_PERSON_NPS)
+        return noun_phrase.replace("[[PERSON]]", person)
+    else:
+        return noun_phrase
+
+
+def make_sentence(template, fields):
+    sentence = template
+    for field, value in fields.items():
+        sentence = sentence.replace(f"[[{field}]]", value)
+    sentence = sentence.replace("_", "").replace("  ", " ").strip()
+    sentence = sentence[0].upper() + sentence[1:]
+    return sentence
+
+
+def main():
+    sentences = {}
+
+    dup_fails = 0
+    while len(sentences) < NUM_SENTENCES and dup_fails < NUM_FAILS:
+        fields = {}
+
+        label = {"party_mentioned": 0, "party_human": 0, "active": False}
+
+        fields["TIME"] = random.choice(TIMES)
+        fields["PLACE"] = random.choice(PLACES)
+
+        sent_type = random.choice(SENT_TYPES)
+        if sent_type == "0_PTY":
+            if random.random() < 0.5:
+                fields["CIRCUMSTANCE"] = random.choice(CIRCUMSTANCES)
+            else:
+                fields["CIRCUMSTANCE"] = random.choice(CIRCUMSTANCES_AGT)
+                label["party_mentioned"] += 1
+                label["party_human"] += 1
+
+            outcome_type = random.choice(OUTCOME_0_TYPES)
+            if outcome_type == "TRAFFIC":
+                fields["OUTCOME"] = random.choice(OUTCOMES_0_TRAFFIC)
+            else:
+                fields["OUTCOME"] = random.choice(OUTCOMES_0_HUMAN)
+                label["party_mentioned"] += 1
+                label["party_human"] += 1
+            sentence = make_sentence(
+                random.choice(SENTS_0_PTY_OUTCOME), fields)
+
+        elif sent_type == "1_PTY":
+            if random.random() < 0.5:
+                fields["CIRCUMSTANCE"] = random.choice(CIRCUMSTANCES)
+            else:
+                fields["CIRCUMSTANCE"] = random.choice(CIRCUMSTANCES_AGT)
+                label["party_mentioned"] += 1
+                label["party_human"] += 1
+
+            sent_subtype = random.choice(SENT_1_PTY_TYPES)
+            if sent_subtype == "VICTIM":
+                label["party_mentioned"] += 1
+                label["party_human"] += 1
+                fields["SUBJECT"] = generate_weak_pty()
+                fields["VERB_V2"] = random.choice(EVENT_VERBS_1_VICTIM)
+                sentence = make_sentence(
+                    random.choice(SENTS_1_PTY_VICTIM), fields)
+            elif sent_subtype == "OUTCOME":
+                subject_type = random.choice(["WEAK_PTY", "DRIVER"])
+                fields["OUTCOME"] = random.choice(OUTCOMES_1)
+                if subject_type == "WEAK_PTY":
+                    label["party_mentioned"] += 1
+                    label["party_human"] += 1
+                    fields["SUBJECT"] = generate_weak_pty()
+                else:  # driver
+                    label["party_mentioned"] += 1
+                    label["party_human"] += 1
+                    fields["SUBJECT"] = generate_driver()
+                sentence = make_sentence(
+                    random.choice(SENTS_1_PTY_OUTCOME), fields)
+            else:  # drive
+                subject_type = random.choice(["DRIVER", "VERHICLE"])
+                active_type = random.choice(SENT_ACTIVE_TYPES)
+                if active_type == "ACTIVE":
+                    fields["VP_DRIVE"] = random.choice(VPS_DRIVE_ACTIVE)
+                    label["active"] = True
+                else:
+                    fields["VP_DRIVE"] = random.choice(VPS_DRIVE_NON_ACTIVE)
+                if subject_type == "DRIVER":
+                    label["party_mentioned"] += 1
+                    label["party_human"] += 1
+                    fields["SUBJECT"] = generate_driver()
+                else:  # vehicle
+                    label["party_mentioned"] += 1
+                    fields["SUBJECT"] = random.choice(VEHICLE_NPS)
+                sentence = make_sentence(
+                    random.choice(SENTS_1_PTY_DRIVE), fields)
+        else:  # 2 pty
+            active_type = random.choice(SENT_ACTIVE_TYPES)
+            if active_type == "ACTIVE":
+                subject_type = random.choice(["WEAK_PTY", "DRIVER", "VERHICLE"])
+                label["active"] = True
+
+                if subject_type == "WEAK_PTY":
+                    label["party_mentioned"] += 1
+                    label["party_human"] += 1
+                    fields["VERB_V2"], fields["VERB_P"], fields["VERB_REST"] = random.choice(
+                        EVENT_VERBS_2_ACTIVE_ANY).split("|")
+                    fields["SUBJECT"] = generate_weak_pty()
+                    other_type = random.choice(["WEAK_PTY", "VEHICLE"])
+                elif subject_type == "DRIVER":
+                    label["party_mentioned"] += 1
+                    label["party_human"] += 1
+                    fields["SUBJECT"] = generate_driver()
+                    if random.random() < 0.5:
+                        fields["VERB_V2"], fields["VERB_P"], fields["VERB_REST"] = random.choice(
+                            EVENT_VERBS_2_ACTIVE_ANY).split("|")
+                        other_type = random.choice(["WEAK_PTY", "VEHICLE"])
+                    else:
+                        fields["VERB_V2"], fields["VERB_P"], fields["VERB_REST"] = random.choice(
+                            EVENT_VERBS_2_ACTIVE_DRIVE).split("|")
+                        other_type = "WEAK_PTY"
+
+                else:  # vehicle
+                    label["party_mentioned"] += 1
+                    fields["VERB_V2"], fields["VERB_P"], fields["VERB_REST"] = random.choice(
+                        EVENT_VERBS_2_ACTIVE_ANY + EVENT_VERBS_2_ACTIVE_DRIVE).split("|")
+                    fields["SUBJECT"] = random.choice(VEHICLE_NPS)
+
+                if other_type == "WEAK_PTY":
+                    label["party_mentioned"] += 1
+                    label["party_human"] += 1
+                    fields["OTHER"] = generate_weak_pty()
+                elif other_type == "DRIVER":
+                    label["party_mentioned"] += 1
+                    label["party_human"] += 1
+                    fields["OTHER"] = generate_driver()
+                else: # vehicle
+                    label["party_mentioned"] += 1
+                    fields["OTHER"] = random.choice(VEHICLE_NPS)
+
+            else:  # non-active
+                other_type = random.choice(["WEAK_PTY", "DRIVER", "VEHICLE"])
+                if other_type == "WEAK_PTY":
+                    label["party_mentioned"] += 1
+                    label["party_human"] += 1
+                    fields["OTHER"] = generate_weak_pty()
+                    fields["VERB_V2"], fields["VERB_P"], fields["VERB_REST"] = random.choice(
+                        EVENT_VERBS_2_NON_ACTIVE_ANY).split("|")
+                    subject_type = random.choice(["WEAK_PTY", "DRIVER", "VEHICLE"])
+
+                elif other_type == "DRIVER":
+                    label["party_mentioned"] += 1
+                    label["party_human"] += 1
+                    fields["OTHER"] = generate_driver()
+                    if random.random() < 0.5:
+                        fields["VERB_V2"], fields["VERB_P"], fields["VERB_REST"] = random.choice(
+                            EVENT_VERBS_2_NON_ACTIVE_ANY).split("|")
+                        subject_type = random.choice(["WEAK_PTY", "DRIVER", "VEHICLE"])
+                    else:
+                        fields["VERB_V2"], fields["VERB_P"], fields["VERB_REST"] = random.choice(
+                            EVENT_VERBS_2_NON_ACTIVE_DRIVER).split("|")
+                        subject_type = random.choice(["WEAK_PTY"])
+
+                else:  # "vehicle"
+                    label["party_mentioned"] += 1
+                    fields["OTHER"] = random.choice(VEHICLE_NPS)
+                    if random.random() < 0.5:
+                        fields["VERB_V2"], fields["VERB_P"], fields["VERB_REST"] = random.choice(
+                            EVENT_VERBS_2_NON_ACTIVE_ANY).split("|")
+                        subject_type = random.choice(["WEAK_PTY", "DRIVER", "VEHICLE"])
+                    else:
+                        fields["VERB_V2"], fields["VERB_P"], fields["VERB_REST"] = random.choice(
+                            EVENT_VERBS_2_NON_ACTIVE_VEHICLE).split("|")
+                        subject_type = random.choice(["WEAK_PTY"])
+
+                if subject_type == "WEAK_PTY":
+                    label["party_mentioned"] += 1
+                    label["party_human"] += 1
+                    fields["SUBJECT"] = generate_weak_pty()
+                elif subject_type == "DRIVER":
+                    label["party_mentioned"] += 1
+                    label["party_human"] += 1
+                    fields["SUBJECT"] = generate_driver()
+                else:  # vehicle
+                    label["party_mentioned"] += 1
+                    fields["SUBJECT"] = random.choice(VEHICLE_NPS)
+
+            sentence = make_sentence(random.choice(SENTS_2_PTYS), fields)
+
+        if sentence not in sentences:
+            sentences[sentence] = label
+            dup_fails = 0
+        else:
+            dup_fails += 1
+
+    with open("output/crashes/generate_templates/sentences.jsonl", "w", encoding="utf-8") as f_out:
+        for sentence, label in sentences.items():
+            f_out.write(json.dumps({"sentence": sentence, "label": label}) + os.linesep)
+        f_out.write(os.linesep)
+
+
+if __name__ == "__main__":
+    main()

sociofillmore/crashes/make_bechdel_dicts.py

@@ -0,0 +1,90 @@
+import sys
+import os
+import json
+
+sys.path.append("./libs")
+from OpenDutchWordnet import Wn_grid_parser
+
+
+def find_all_le_hyponyms(instance, le_id):
+    print(f"Starting from `{le_id}`...")
+    le_el = instance.les_find_le(le_id)
+    le_ss = le_el.get_synset_id()
+    siblings = {le.get_id() for le in instance.les_all_les_of_one_synset(le_ss)}
+    print(f"Siblings: {siblings}")
+    synset_el = instance.synsets_find_synset(le_ss)
+    print(f"Top-level synset: `{le_el.get_synset_id()}`...")
+    hyponyms = find_all_synset_hyponyms(instance, synset_el)
+    return siblings.union(hyponyms)
+    
+
+def find_all_synset_hyponyms(instance, synset_el):
+    print(f"Finding hyponyms of synset with gloss: `{synset_el.get_glosses()[:1]}`...")
+    hypo_les = set()
+    hypo_rels = synset_el.get_relations("has_hyponym")
+    for rel in hypo_rels:
+        hypo_ss = rel.get_target()
+        print(hypo_ss)
+        ss_les = {le.get_id() for le in instance.les_all_les_of_one_synset(hypo_ss)}
+        for i in ss_les:
+            print(f"\tfound LE: {i}")
+        ss_les.update(find_all_synset_hyponyms(instance, instance.synsets_find_synset(hypo_ss)))
+        hypo_les.update(ss_les)
+    return hypo_les
+
+
+def find_siblings_and_hyperonym(instance, le_id):
+    le_el = instance.les_find_le(le_id)
+    le_ss = le_el.get_synset_id()
+    siblings = {le.get_id() for le in instance.les_all_les_of_one_synset(le_ss)}
+    print(siblings)
+    synset_el = instance.synsets_find_synset(le_ss)
+    hyper = synset_el.get_relations("has_hyperonym")[0]
+    hyper_ss = instance.synsets_find_synset(hyper.get_target())
+    print(hyper_ss.get_glosses())
+    print({le.get_id() for le in instance.les_all_les_of_one_synset(hyper.get_target())})
+
+
+def main():
+    instance = Wn_grid_parser(Wn_grid_parser.odwn)
+    # find_all_le_hyponyms(instance, "slachtoffer-n-4")
+    dicts = {
+        "vehicles": {
+            "WN:cars": sorted(find_all_le_hyponyms(instance, "automobiel-n-1")),
+            "WN:motorbikes": sorted(find_all_le_hyponyms(instance, "motorfiets-n-1")),
+            "WN:bikes": sorted(find_all_le_hyponyms(instance, "fiets-n-1")),
+            "WN:buses": sorted(find_all_le_hyponyms(instance, "autobus-n-1")),
+            "extra": sorted(["scootmobiel", "e-bike"])
+        },
+        "persons": {
+            "WN:driver": sorted(find_all_le_hyponyms(instance, "bestuurder-n-2")),
+            "WN:cyclist": sorted(find_all_le_hyponyms(instance, "fietser-n-1")),
+            "WN:walker": sorted(find_all_le_hyponyms(instance, "loper-n-4")),
+            "WN:pedestrian": sorted(find_all_le_hyponyms(instance, "voetganger-n-1")),
+            "WN:victim": sorted(find_all_le_hyponyms(instance, "slachtoffer-n-4")),
+            "extra": sorted(
+                ["man", "vrouw", "jongen", "meisje", "persoon", "bejaarde", "maaltijdbezorger"]
+                )
+        }
+    }
+
+    ignore_file = "output/crashes/predict_bechdel/lexical_dicts_ignore.json"
+    if os.path.isfile(ignore_file):
+        with open(ignore_file, encoding="utf-8") as f_ign:
+            ignore = json.load(f_ign)
+        
+        cleaned_dicts = {}
+        for category in dicts.keys():
+            cleaned_dicts[category] = {}
+            for subcat, words in dicts[category].items():
+                ignore_subcat = ignore.get(category, {}).get(subcat, [])
+                cleaned_dicts[category][subcat] = [w for w in words if w not in ignore_subcat]
+    else:
+        cleaned_dicts = dicts
+
+    with open("output/crashes/predict_bechdel/lexical_dicts.json", "w", encoding="utf-8") as f_out:
+        json.dump(cleaned_dicts, f_out, indent=4) 
+
+
+if __name__ == "__main__":
+    main()

sociofillmore/crashes/predict_bechdel.py

@@ -0,0 +1,500 @@
+import os
+import re
+from typing import Dict, Iterable, List, Optional, Tuple
+import json
+import random
+import argparse
+from allennlp.data.fields.field import Field
+from allennlp.data.fields.sequence_field import SequenceField
+from allennlp.models.model import Model
+from allennlp.nn.util import get_text_field_mask
+from allennlp.predictors.predictor import Predictor
+
+import pandas as pd
+import spacy
+import torch
+from sklearn.preprocessing import MultiLabelBinarizer
+
+from allennlp.common.util import pad_sequence_to_length
+from allennlp.data import TextFieldTensors
+from allennlp.data.vocabulary import Vocabulary
+from allennlp.data import DatasetReader, TokenIndexer, Instance, Token
+from allennlp.data.fields import TextField, LabelField
+from allennlp.data.token_indexers.pretrained_transformer_indexer import (
+    PretrainedTransformerIndexer,
+)
+from allennlp.data.tokenizers.pretrained_transformer_tokenizer import (
+    PretrainedTransformerTokenizer,
+)
+from allennlp.models import BasicClassifier
+from allennlp.modules.text_field_embedders.basic_text_field_embedder import (
+    BasicTextFieldEmbedder,
+)
+from allennlp.modules.token_embedders.pretrained_transformer_embedder import (
+    PretrainedTransformerEmbedder,
+)
+from allennlp.modules.seq2vec_encoders.bert_pooler import BertPooler
+from allennlp.modules.seq2vec_encoders.cls_pooler import ClsPooler
+from allennlp.training.checkpointer import Checkpointer
+from allennlp.training.gradient_descent_trainer import GradientDescentTrainer
+from allennlp.data.data_loaders.simple_data_loader import SimpleDataLoader
+from allennlp.training.optimizers import AdamOptimizer
+from allennlp.predictors.text_classifier import TextClassifierPredictor
+from allennlp.training.callbacks.tensorboard import TensorBoardCallback
+from torch import nn
+from torch.nn.functional import binary_cross_entropy_with_logits
+
+
+random.seed(1986)
+
+
+SEQ_LABELS = ["humansMentioned", "vehiclesMentioned", "eventVerb", "activeEventVerb"]
+
+
+# adapted from bert-for-framenet project
+class SequenceMultiLabelField(Field):
+
+    def __init__(self,
+                 labels: List[List[str]],
+                 sequence_field: SequenceField,
+                 binarizer: MultiLabelBinarizer,
+                 label_namespace: str
+                 ):
+        self.labels = labels
+        self._indexed_labels = None
+        self._label_namespace = label_namespace
+        self.sequence_field = sequence_field
+        self.binarizer = binarizer
+
+    @staticmethod
+    def retokenize_tags(tags: List[List[str]],
+                    offsets: List[Tuple[int, int]],
+                    wp_primary_token: str = "last",
+                    wp_secondary_tokens: str = "empty",
+                    empty_value=lambda: []
+                    ) -> List[List[str]]:
+        tags_per_wordpiece = [
+            empty_value()  # [CLS]
+        ]
+
+        for i, (off_start, off_end) in enumerate(offsets):
+            tag = tags[i]
+
+            # put a tag on the first wordpiece corresponding to the word token
+            # e.g. "hello" --> "he" + "##ll" + "##o" --> 2 extra tokens
+            # TAGS: [..., TAG, None, None, ...]
+            num_extra_tokens = off_end - off_start
+            if wp_primary_token == "first":
+                tags_per_wordpiece.append(tag)
+            if wp_secondary_tokens == "repeat":
+                tags_per_wordpiece.extend(num_extra_tokens * [tag])
+            else:
+                tags_per_wordpiece.extend(num_extra_tokens * [empty_value()])
+            if wp_primary_token == "last":
+                tags_per_wordpiece.append(tag)
+
+        tags_per_wordpiece.append(empty_value())  # [SEP]
+
+        return tags_per_wordpiece
+
+    def count_vocab_items(self, counter: Dict[str, Dict[str, int]]):
+        for label_list in self.labels:
+            for label in label_list:
+                counter[self._label_namespace][label] += 1
+
+    def get_padding_lengths(self) -> Dict[str, int]:
+        return {"num_tokens": self.sequence_field.sequence_length()}
+
+    def index(self, vocab: Vocabulary):
+
+        indexed_labels: List[List[int]] = []
+        for sentence_labels in self.labels:
+            sentence_indexed_labels = []
+            for label in sentence_labels:
+                try:
+                    sentence_indexed_labels.append(
+                        vocab.get_token_index(label, self._label_namespace))
+                except KeyError:
+                    print(f"[WARNING] Ignore unknown label {label}")
+            indexed_labels.append(sentence_indexed_labels)
+        self._indexed_labels = indexed_labels
+
+    def as_tensor(self, padding_lengths: Dict[str, int]) -> torch.Tensor:
+
+        # binarize
+        binarized_seq = self.binarizer.transform(self._indexed_labels).tolist()
+
+        # padding
+        desired_num_tokens = padding_lengths["num_tokens"]
+        padded_tags = pad_sequence_to_length(binarized_seq, desired_num_tokens,
+                                             default_value=lambda: list(self.binarizer.transform([[]])[0]))
+
+        tensor = torch.tensor(padded_tags, dtype=torch.float)
+        return tensor
+
+    def empty_field(self) -> 'Field':
+
+        field = SequenceMultiLabelField(
+            [], self.sequence_field.empty_field(), self.binarizer, self._label_namespace)
+        field._indexed_labels = []
+        return field
+
+
+# adapted from bert-for-framenet project
+class MultiSequenceLabelModel(Model):
+
+    def __init__(self, embedder: PretrainedTransformerEmbedder, decoder_output_size: int, hidden_size: int, vocab: Vocabulary, embedding_size: int = 768):
+        super().__init__(vocab)
+        self.embedder = embedder
+        self.out_features = decoder_output_size
+        self.hidden_size = hidden_size
+        self.layers = nn.Sequential(
+            nn.Linear(in_features=embedding_size,
+                      out_features=self.hidden_size),
+            nn.ReLU(),
+            nn.Linear(in_features=self.hidden_size,
+                      out_features=self.out_features)
+        )
+
+    def forward(self, tokens: TextFieldTensors, label: Optional[torch.FloatTensor] = None):
+        embeddings = self.embedder(tokens["token_ids"])
+        mask = get_text_field_mask(tokens).float()
+        tag_logits = self.layers(embeddings)
+        mask = mask.reshape(mask.shape[0], mask.shape[1], 1).repeat(1, 1, self.out_features)
+        output = {"tag_logits": tag_logits}
+        if label is not None:
+            loss = binary_cross_entropy_with_logits(tag_logits, label, mask)
+            output["loss"] = loss
+
+    def get_metrics(self, _) -> Dict[str, float]:
+        return {}
+
+    def make_human_readable(self,
+                                  prediction,
+                                  label_namespace,
+                                  threshold=0.2,
+                                  sigmoid=True
+                                  ) -> Tuple[List[str], Optional[List[float]]]:
+        if sigmoid:
+            prediction = torch.sigmoid(prediction)
+
+        predicted_labels: List[List[str]] = [[] for _ in range(len(prediction))]
+
+        # get all predictions with a positive probability
+        for coord in torch.nonzero(prediction > threshold):
+            label = self.vocab.get_token_from_index(int(coord[1]), label_namespace)
+            predicted_labels[coord[0]].append(f"{label}:{prediction[coord[0], coord[1]]:.3f}")
+
+        str_predictions: List[str] = []
+        for label_list in predicted_labels:
+            str_predictions.append("|".join(label_list) or "_")
+
+        return str_predictions
+
+
+class TrafficBechdelReader(DatasetReader):
+
+    def __init__(self, token_indexers, tokenizer, binarizer):
+        self.token_indexers = token_indexers
+        self.tokenizer: PretrainedTransformerTokenizer = tokenizer
+        self.binarizer = binarizer
+        self.orig_data = []
+        super().__init__()
+
+    def _read(self, file_path) -> Iterable[Instance]:
+        self.orig_data.clear()
+
+        with open(file_path, encoding="utf-8") as f:
+            for line in f:
+                # skip any empty lines
+                if not line.strip():
+                    continue
+
+                sentence_parts = line.lstrip("[").rstrip("]").split(",")
+                token_txts = []
+                token_mlabels = []
+                
+                for sp in sentence_parts:
+                    sp_txt, sp_lbl_str = sp.split(":")
+                    if sp_lbl_str == "[]":
+                        sp_lbls = []
+                    else:
+                        sp_lbls = sp_lbl_str.lstrip("[").rstrip("]").split("|")
+                    
+                    # if the text is a WordNet thingy
+                    wn_match = re.match(r"^(.+)-n-\d+$", sp_txt)
+                    if wn_match:
+                        sp_txt = wn_match.group(1)
+                    
+                    # multi-token text
+                    sp_toks = sp_txt.split()
+                    for tok in sp_toks:
+                        token_txts.append(tok)
+                        token_mlabels.append(sp_lbls)
+                
+                self.orig_data.append({
+                    "sentence": token_txts,
+                    "labels": token_mlabels,
+                })
+                yield self.text_to_instance(token_txts, token_mlabels)
+
+    def text_to_instance(self, sentence: List[str], labels: List[List[str]] = None) -> Instance:
+        tokens, offsets = self.tokenizer.intra_word_tokenize(sentence)
+
+        text_field = TextField(tokens, self.token_indexers)
+        fields = {"tokens": text_field}
+        if labels is not None:
+            labels_ = SequenceMultiLabelField.retokenize_tags(labels, offsets)
+            label_field = SequenceMultiLabelField(labels_, text_field, self.binarizer, "labels")
+            fields["label"] = label_field
+        return Instance(fields)
+
+
+def count_parties(sentence, lexical_dicts, nlp):
+
+    num_humans = 0
+    num_vehicles = 0
+
+    def is_in_words(l, category):
+        for subcategory, words in lexical_dicts[category].items():
+            if subcategory.startswith("WN:"):
+                words = [re.match(r"^(.+)-n-\d+$", w).group(1) for w in words]
+            if l in words:
+                return True
+        return False
+
+    doc = nlp(sentence.lower())
+    for token in doc:
+        lemma = token.lemma_
+        if is_in_words(lemma, "persons"):
+            num_humans += 1
+        if is_in_words(lemma, "vehicles"):
+            num_vehicles += 1
+
+    return num_humans, num_vehicles
+
+
+def predict_rule_based(annotations="data/crashes/bechdel_annotations_dev_first_25.csv"):
+    data_crashes = pd.read_csv(annotations)
+    with open("output/crashes/predict_bechdel/lexical_dicts.json", encoding="utf-8") as f:
+        lexical_dicts = json.load(f)
+
+    nlp = spacy.load("nl_core_news_md")
+
+    for _, row in data_crashes.iterrows():
+        sentence = row["sentence"]
+        num_humans, num_vehicles = count_parties(sentence, lexical_dicts, nlp)
+        print(sentence)
+        print(f"\thumans={num_humans}, vehicles={num_vehicles}")
+
+
+def evaluate_crashes(predictor, attrib, annotations="data/crashes/bechdel_annotations_dev_first_25.csv", out_file="output/crashes/predict_bechdel/predictions_crashes25.csv"):
+    data_crashes = pd.read_csv(annotations)
+    labels_crashes = [
+        {
+            "party_mentioned": str(row["mentioned"]),
+            "party_human": str(row["as_human"]),
+            "active": str(True) if str(row["active"]).lower() == "true" else str(False)
+        }
+        for _, row in data_crashes.iterrows()
+    ]
+    predictions_crashes = [predictor.predict(
+        row["sentence"]) for i, row in data_crashes.iterrows()]
+    crashes_out = []
+    correct = 0
+    partial_2_attrs = 0
+    partial_1_attr = 0
+    correct_mentions = 0
+    correct_humans = 0
+    correct_active = 0
+
+    for sentence, label, prediction in zip(data_crashes["sentence"], labels_crashes, predictions_crashes):
+        predicted = prediction["label"]
+        if attrib == "all":
+            gold = "|".join([f"{k}={v}" for k, v in label.items()])
+        else:
+            gold = label["attrib"]
+        if gold == predicted:
+            correct += 1
+            if attrib == "all":
+                partial_2_attrs += 1
+                partial_1_attr += 1
+
+        if attrib == "all":
+            gold_attrs = set(gold.split("|"))
+            pred_attrs = set(predicted.split("|"))
+            if len(gold_attrs & pred_attrs) == 2:
+                partial_2_attrs += 1
+                partial_1_attr += 1
+            elif len(gold_attrs & pred_attrs) == 1:
+                partial_1_attr += 1
+
+            if gold.split("|")[0] == predicted.split("|")[0]:
+                correct_mentions += 1
+            if gold.split("|")[1] == predicted.split("|")[1]:
+                correct_humans += 1
+            if gold.split("|")[2] == predicted.split("|")[2]:
+                correct_active += 1
+
+        crashes_out.append(
+            {"sentence": sentence, "gold": gold, "prediction": predicted})
+
+    print("ACC_crashes (strict) = ", correct/len(data_crashes))
+    print("ACC_crashes (partial:2) = ", partial_2_attrs/len(data_crashes))
+    print("ACC_crashes (partial:1) = ", partial_1_attr/len(data_crashes))
+    print("ACC_crashes (mentions) = ", correct_mentions/len(data_crashes))
+    print("ACC_crashes (humans) = ", correct_humans/len(data_crashes))
+    print("ACC_crashes (active) = ", correct_active/len(data_crashes))
+
+    pd.DataFrame(crashes_out).to_csv(out_file)
+
+
+def filter_events_for_bechdel():
+
+    with open("data/crashes/thecrashes_data_all_text.json", encoding="utf-8") as f:
+        events = json.load(f)
+
+    total_articles = 0
+    data_out = []
+    for ev in events:
+        total_articles += len(ev["articles"])
+
+        num_persons = len(ev["persons"])
+        num_transport_modes = len({p["transportationmode"]
+                                  for p in ev["persons"]})
+
+        if num_transport_modes <= 2:
+            for art in ev["articles"]:
+                data_out.append({"event_id": ev["id"], "article_id": art["id"], "headline": art["title"],
+                                "num_persons": num_persons, "num_transport_modes": num_transport_modes})
+
+    print("Total articles = ", total_articles)
+
+    print("Filtered articles: ", len(data_out))
+    out_df = pd.DataFrame(data_out)
+    out_df.to_csv("output/crashes/predict_bechdel/filtered_headlines.csv")
+
+
+def train_and_eval(train=True):
+
+    # use_gpu = False
+    use_gpu = True
+    cuda_device = None if use_gpu and torch.cuda.is_available() else -1
+
+    transformer = "GroNLP/bert-base-dutch-cased"
+    # transformer = "xlm-roberta-large"
+    token_indexers = {"tokens": PretrainedTransformerIndexer(transformer)}
+    tokenizer = PretrainedTransformerTokenizer(transformer)
+
+    binarizer = MultiLabelBinarizer()
+    binarizer.fit([SEQ_LABELS])
+    reader = TrafficBechdelReader(token_indexers, tokenizer, binarizer)
+    instances = list(reader.read("output/prolog/bechdel_headlines.txt"))
+    orig_data = reader.orig_data
+    zipped = list(zip(instances, orig_data))
+    random.shuffle(zipped)
+    instances_ = [i[0] for i in zipped]
+    orig_data_ = [i[1] for i in zipped]
+
+    num_dev = round(0.05 * len(instances_))
+    num_test = round(0.25 * len(instances_))
+    num_train = len(instances_) - num_dev - num_test
+    print("LEN(train/dev/test)=", num_train, num_dev, num_test)
+
+    instances_train = instances_[:num_train]
+    instances_dev = instances_[num_train:num_train + num_dev]
+    # instances_test = instances_[num_train+num_dev:num_train:]
+
+    # orig_train = orig_data_[:num_train]
+    orig_dev = orig_data_[num_train:num_train + num_dev]
+
+    vocab = Vocabulary.from_instances(instances_train + instances_dev)
+
+    embedder = BasicTextFieldEmbedder(
+        {"tokens": PretrainedTransformerEmbedder(transformer)})
+    model = MultiSequenceLabelModel(embedder, len(SEQ_LABELS), 1000, vocab)
+    if use_gpu:
+        model = model.cuda(cuda_device)
+
+    # checkpoint_dir = f"output/crashes/predict_bechdel/model_{attrib}/"
+    checkpoint_dir = f"/scratch/p289731/predict_bechdel/model_seqlabel/"
+    serialization_dir = f"/scratch/p289731/predict_bechdel/serialization_seqlabel/"
+
+    if train:
+        os.makedirs(checkpoint_dir)
+        os.makedirs(serialization_dir)
+        tensorboard = TensorBoardCallback(
+            serialization_dir, should_log_learning_rate=True)
+        checkpointer = Checkpointer(serialization_dir=checkpoint_dir)
+        optimizer = AdamOptimizer(
+            [(n, p) for n, p in model.named_parameters() if p.requires_grad],
+            lr=1e-5
+        )
+        train_loader = SimpleDataLoader(
+            instances_train, batch_size=8, shuffle=True)
+        dev_loader = SimpleDataLoader(
+            instances_dev, batch_size=8, shuffle=False)
+        train_loader.index_with(vocab)
+        dev_loader.index_with(vocab)
+
+        print("\t\tTraining BERT model")
+        trainer = GradientDescentTrainer(
+            model,
+            optimizer,
+            train_loader,
+            validation_data_loader=dev_loader,
+            # patience=32,
+            patience=2,
+            # num_epochs=1,
+            checkpointer=checkpointer,
+            cuda_device=cuda_device,
+            serialization_dir=serialization_dir,
+            callbacks=[tensorboard]
+        )
+        trainer.train()
+    else:
+        state_dict = torch.load(
+            "/scratch/p289731/predict_bechdel/serialization_all/best.th", map_location=cuda_device)
+        model.load_state_dict(state_dict)
+
+    print("\t\tProducing predictions...")
+
+    predictor = Predictor(model, reader)
+    predictions_dev = [predictor.predict_instance(i) for i in instances_dev]
+
+    data_out = []
+    for sentence, prediction in zip(orig_dev, predictions_dev):
+        readable = model.make_human_readable(prediction, "labels")
+        text = sentence["sentence"]
+        gold = sentence["labels"]
+        predicted = readable
+        data_out.append(
+            {"sentence": text, "gold": gold, "predicted": predicted})
+    df_out = pd.DataFrame(data_out)
+    df_out.to_csv("output/crashes/predict_bechdel/predictions_dev.csv")
+
+    # print()
+
+    # print("First 25 crashes:")
+    # evaluate_crashes(predictor, attrib, annotations="data/crashes/bechdel_annotations_dev_first_25.csv",
+    #                  out_file="output/crashes/predict_bechdel/predictions_first_25.csv")
+    # print()
+    # print("Next 75 crashes:")
+    # evaluate_crashes(predictor, attrib, annotations="data/crashes/bechdel_annotations_dev_next_75.csv",
+    #                  out_file="output/crashes/predict_bechdel/predictions_next_75.csv")
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("action", choices=["train", "predict", "rules", "filter"])
+
+    args = ap.parse_args()
+
+    if args.action == "train":
+        train_and_eval(train=True)
+    elif args.action == "predict":
+        train_and_eval(train=False)
+    elif args.action == "rules":
+        predict_rule_based()
+    else:
+        filter_events_for_bechdel()

sociofillmore/crashes/split_data.py

@@ -0,0 +1,240 @@
+import json
+import os
+import random
+import shutil
+from datetime import datetime
+
+import langdetect
+import nltk
+import pandas as pd
+from langdetect import DetectorFactory, LangDetectException
+
+DATA_FILE = "data/crashes/thecrashes_data_all_text.json"
+
+DEV_PORTION = .10
+
+random.seed(2001)
+DetectorFactory.seed = 0
+
+
+def is_a_real_time(timestamp):
+    """Helper function, checks if a given timestamp really has a time"""
+
+    # 00:00:00 (midnight) is the "empty" timestamp, ignore it
+    if timestamp.hour == timestamp.minute == timestamp.second == 0:
+        return False
+    return True
+
+
+def main():
+    process_events()
+
+
+def detect_language(article):
+    if article["alltext"]:
+        sample = article["alltext"]
+    elif article["summary"]:
+        sample = article["summary"]
+    else:
+        sample = article["title"]
+
+    try:
+        return langdetect.detect(sample)
+    except LangDetectException:
+        print(f"\tCould not detect language for text_id={article['id']}")
+        print(f"\tSample={sample})")
+        print()
+        return "UNK_LANG"
+
+
+def extract_text_info(event):
+    ev_text_lines = []
+    ev_id_lines = []
+    ev_meta_rows = []
+
+    for article in event["articles"]:
+        text_id = article["id"]
+        try:
+            pubdate = datetime.fromisoformat(article["publishedtime"]).strftime("%Y-%m-%d %H:%M:%S")
+        except ValueError:
+            print(f"\t\tcould not parse date {article['publishedtime']}")
+            pubdate = None
+        url = article["url"]
+        provider = article["sitename"]
+        title = article["title"]
+        language = detect_language(article)
+        ev_meta_rows.append({
+            "event_id": event["id"],
+            "text_id": text_id,
+            "pubdate": pubdate,
+            "language": language,
+            "url": url,
+            "provider": provider,
+            "title": title
+        })
+
+        summary = article["summary"]
+        body = article["alltext"]
+
+        text_lines = []
+        id_lines = []
+
+        for line in segment(title, language):
+            text_lines.append(line)
+            id_lines.append(f"event {event['id']}\ttext {text_id}\ttitle")
+
+        for line in segment(summary, language):
+            text_lines.append(line)
+            id_lines.append(f"event {event['id']}\ttext {text_id}\tsummary")
+
+        for line in segment(body, language):
+            text_lines.append(line)
+            id_lines.append(f"event {event['id']}\ttext {text_id}\tbody")
+
+        ev_text_lines.append(text_lines)
+        ev_id_lines.append(id_lines)
+
+    return ev_text_lines, ev_id_lines, ev_meta_rows
+
+
+def segment(text, language):
+    # don't split Hebrew and Vietnamese (because we don't have a segmenter for it)
+    if language in ["he", "vi"]:
+        return text
+
+    lang_map = {
+        "nl": "dutch",
+        "en": "english",
+        "es": "spanish",
+        "de": "german",
+        "fr": "french",
+        "ru": "russian",
+        "pt": "portuguese"
+    }
+
+    nltk_lang = lang_map.get(language)
+
+    # what to do with languages without sent tokenizer in NLTK (apart from Hebrew):
+    if not nltk_lang:
+        if language == "af":
+            # treat Afrikaans as Dutch
+            nltk_lang = "dutch"
+        else:
+            print(f"Found an article with unsupported language={language}, falling back to English NLTK")
+            nltk_lang = "english"
+
+    return nltk.sent_tokenize(text, nltk_lang)
+
+
+def write_to_text_by_event(text_lines, text_meta_lines, event_id, split_to_dir, split):
+    event_dir = f"{split_to_dir[split]}/{event_id}"
+    os.makedirs(event_dir, exist_ok=True)
+    for art_lines, row in zip(text_lines, text_meta_lines):
+        text_file = f"{event_dir}/{row['text_id']}.txt"
+        with open(text_file, "w", encoding="utf-8") as f:
+            for line in art_lines:
+                f.write(line + os.linesep)
+
+
+def process_events():
+    print("Loading data file...")
+    with open(DATA_FILE, encoding="utf-8") as f:
+        data = json.load(f)
+    event_all_rows = []
+    event_dev_rows = []
+    event_main_rows = []
+
+    text_all_rows = []
+    text_dev_rows = []
+    text_main_rows = []
+
+    # make empty text files
+    text_file_basenames = {
+        "all": "output/crashes/split_data/all.texts",
+        "dev": "output/crashes/split_data/split_dev10.texts",
+        "main": "output/crashes/split_data/split_main.texts"
+    }
+    for split, bn in text_file_basenames.items():
+        for ext in [".text.txt", ".ids.txt"]:
+            f = open(f"{bn}{ext}", "w", encoding="utf-8")
+            f.close()
+
+    # clear & make text file directories
+    text_files_by_event_dir = {}
+    for split in ["all", "dev", "main"]:
+        prefix = "split_dev10" if split == "dev" else "split_main" if split == "main" else "all"
+        text_dir = f"output/crashes/split_data/{prefix}_texts_by_event"
+        text_files_by_event_dir[split] = text_dir
+        if os.path.exists(text_dir):
+            shutil.rmtree(text_dir)
+        os.mkdir(text_dir)
+
+    # helper function for writing text files
+    def append_to_txt(txt_file, lines):
+        with open(txt_file, "a", encoding="utf-8") as f_out:
+            for art_lines in lines:
+                for line in art_lines:
+                    f_out.write(line + os.linesep)
+
+    print("Processing events...")
+    for event in data:
+        event_id = event["id"]
+        print(f"\tevent_id={event_id}")
+        try:
+            timestamp = datetime.fromisoformat(event["date"])
+        except ValueError:
+            timestamp = None
+
+        event_row = {
+            "event:id": event_id,
+            "event:date": timestamp.strftime("%Y-%m-%d") if timestamp else None,
+            "event:time": timestamp.strftime("%H-%M-%S") if timestamp and is_a_real_time(timestamp) else None,
+            "event:coordinates": f"{event['latitude'], event['longitude']}",
+            "vehicle_involved": 1 if any(p for p in event["persons"] if p["transportationmode"] in range(5, 14)) else 0
+        }
+
+        for health, health_code in (("dead", 3), ("injured", 2)):
+            all_with_health = [p for p in event["persons"] if p["health"] == health_code]
+            event_row[f"outcomes:{health}:total"] = len(all_with_health)
+            event_row[f"outcomes:{health}:child"] = len([p for p in all_with_health if p["child"] == 1])
+            for mode, mode_codes in (("pedestrian", [1]), ("cyclist", [2]), ("vehicle", range(5, 14))):
+                event_row[f"outcomes:{health}:{mode}"] = len([p for p in all_with_health
+                                                              if p["transportationmode"] in mode_codes])
+
+        text_lines, text_id_lines, text_meta_rows = extract_text_info(event)
+
+        event_all_rows.append(event_row)
+        text_all_rows.extend(text_meta_rows)
+        append_to_txt(text_file_basenames["all"] + ".text.txt", text_lines)
+        append_to_txt(text_file_basenames["all"] + ".ids.txt", text_id_lines)
+        write_to_text_by_event(text_lines, text_meta_rows, event_id, text_files_by_event_dir, "all")
+
+        if random.random() < DEV_PORTION:
+            event_dev_rows.append(event_row)
+            text_dev_rows.extend(text_meta_rows)
+            append_to_txt(text_file_basenames["dev"] + ".text.txt", text_lines)
+            append_to_txt(text_file_basenames["dev"] + ".ids.txt", text_id_lines)
+            write_to_text_by_event(text_lines, text_meta_rows, event_id, text_files_by_event_dir, "dev")
+
+        else:
+            event_main_rows.append(event_row)
+            text_main_rows.extend(text_meta_rows)
+            append_to_txt(text_file_basenames["main"] + ".text.txt", text_lines)
+            append_to_txt(text_file_basenames["main"] + ".ids.txt", text_id_lines)
+            write_to_text_by_event(text_lines, text_meta_rows, event_id, text_files_by_event_dir, "main")
+
+    all_ev_df = pd.DataFrame(event_all_rows)
+    main_ev_df = pd.DataFrame(event_main_rows)
+    dev_ev_df = pd.DataFrame(event_dev_rows)
+    for df, file in ((all_ev_df, "all.events"), (main_ev_df, "split_main.events"), (dev_ev_df, "split_dev10.events")):
+        df.to_csv(f"output/crashes/split_data/{file}.csv")
+
+    all_txt_df = pd.DataFrame(text_all_rows)
+    main_txt_df = pd.DataFrame(text_main_rows)
+    dev_txt_df = pd.DataFrame(text_dev_rows)
+    for df, file in ((all_txt_df, "all.texts"), (main_txt_df, "split_main.texts"), (dev_txt_df, "split_dev10.texts")):
+        df.to_csv(f"output/crashes/split_data/{file}.meta.csv")
+
+
+if __name__ == '__main__':
+    main()

sociofillmore/crashes/utils.py

@@ -0,0 +1,16 @@
+import pandas as pd
+
+texts_meta = pd.read_csv("output/crashes/split_data/split_dev10.texts.meta.csv", index_col=0, dtype={"text_id": str})
+
+
+def is_a_dutch_text(doc_id, exclude_frisian=True):
+    filtered_for_doc = texts_meta[texts_meta["text_id"] == doc_id]
+    if len(filtered_for_doc) >= 1:
+        if exclude_frisian:
+            # exclude newsproviders publishing mainly in Frisian
+            # (NB these texts are recognized as Dutch by langdetect, hence the need for a provider filter)
+            if filtered_for_doc["provider"].iloc[0] == "omropfryslan.nl":
+                return False
+        if filtered_for_doc["language"].iloc[0] == "nl":
+            return True
+    return False

sociofillmore/femicides/compare_lome_models.py

@@ -0,0 +1,296 @@
+from collections import defaultdict
+import glob
+import json
+import os
+import re
+import random
+import sys
+from typing import List, Dict, Tuple
+
+import pandas as pd
+import numpy as np
+
+from sociofillmore.common.analyze_text import load_caches, process_fn_sentence, FrameStructure, read_frames_of_interest
+
+RANDOM_SEED = 9718
+NUM_EVALUATION_SENTENCES = 150
+
+EVALITA_MODEL = "lome_evalita_plus_fn"
+# EVALITA_MODEL = "lome_evalita_plus_fn_0conf"
+OUT_FOLDER = f"0shot__vs__{EVALITA_MODEL.split('_', maxsplit=1)[1]}"
+print(OUT_FOLDER)
+
+
+random.seed(RANDOM_SEED)
+
+
+def map_predicates_to_frames(structures: List[FrameStructure]) -> Dict[str, str]:
+    mapping = {}
+    for struct in structures:
+        pred_key = "_".join(struct.target.tokens_str)
+        mapping[pred_key] = struct.frame
+    return mapping
+
+
+def make_evaluation_sample(diffs_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
+
+    def make_experimental_columns(row: pd.Series):
+        if random.choice((True, False)):
+            left_col = "predicted_frame_0shot"
+            right_col = "predicted_frame_evalita"
+        else:
+            left_col = "predicted_frame_evalita"
+            right_col = "predicted_frame_0shot"
+
+        exp_info = pd.Series({
+            "prediction_1": row[left_col],
+            "prediction_2": row[right_col],
+            "model_1": left_col,
+            "model_2": right_col
+        })
+
+        return row.append(exp_info)
+
+    sample = diffs_df.sample(n=NUM_EVALUATION_SENTENCES,
+                             random_state=RANDOM_SEED).reset_index(drop=True)
+    with_exp_info = sample.apply(make_experimental_columns, axis=1)
+    annotator_sheet = with_exp_info[[
+        "sentence", "predicate", "prediction_1", "prediction_2"]]
+    # add answer columns
+    for answer_field in ["1_is_best", "2_is_best", "both_are_good", "both_are_bad", "missing_frame"]:
+        annotator_sheet.insert(len(annotator_sheet.columns),
+                               f"answer::{answer_field}", np.nan)
+        # annotator_sheet[f"answer::{answer_field}"] = np.nan
+
+    return annotator_sheet, with_exp_info
+
+
+def make_annotation_experiment():
+    _, deep_frame_cache = load_caches("femicides/rai")
+    frames_of_interest = read_frames_of_interest("femicides/rai")
+
+    all_differences = []
+    foi_differences = []  # foi='frame of interest'
+
+    # number of predicates that have been annotated by at least one model
+    num_all_predictions = 0
+    num_foi_predictions = 0
+
+    num_z_shot_all_predictions = 0
+    num_z_shot_foi_predictions = 0
+
+    num_evalita_all_predictions = 0
+    num_evalita_foi_predictions = 0
+
+    for ev_dir in sorted(glob.glob("output/femicides/lome/lome_0shot/multilabel/rai/*")):
+        ev_id = os.path.basename(ev_dir).rstrip("/")
+        print(f"event={ev_id}")
+        for doc_file in sorted(glob.glob(f"{ev_dir}/*.comm.json")):
+            doc_id = re.search(r'/lome_(\d+)\.comm\.json', doc_file).group(1)
+            print(f"\tdoc={doc_id}")
+
+            with open(doc_file, encoding="utf-8") as f:
+                z_shot_annotations = json.load(f)
+
+            with open(doc_file.replace("/lome_0shot/", f"/{EVALITA_MODEL}/"), encoding="utf-8") as f:
+                evalita_annotations = json.load(f)
+
+            for sent_idx, (z_shot_sent, evalita_sent) in enumerate(zip(z_shot_annotations, evalita_annotations)):
+                z_shot_structs = process_fn_sentence(
+                    z_shot_sent, deep_frame_cache)
+                evalita_structs = process_fn_sentence(
+                    evalita_sent, deep_frame_cache)
+
+                z_shot_frames = {s.frame for s in z_shot_structs.values()}
+                evalita_frames = {s.frame for s in evalita_structs.values()}
+                overlapping_frames = z_shot_frames.intersection(evalita_frames)
+
+                print(f"\t\tsent #{sent_idx}: {len(z_shot_frames)}x lome_0shot frames, "
+                      f"{len(evalita_frames)}x evalita frames, {len(overlapping_frames)}x overlapping")
+
+                z_shot_preds_to_frames = map_predicates_to_frames(
+                    z_shot_structs.values())
+                evalita_preds_to_frames = map_predicates_to_frames(
+                    evalita_structs.values())
+                all_predicates = sorted(set(z_shot_preds_to_frames.keys()).union(
+                    evalita_preds_to_frames.keys()))
+
+                for predicate in all_predicates:
+                    print(f"\t\t\tpredicate={predicate}")
+                    z_shot_frame = z_shot_preds_to_frames.get(predicate)
+                    evalita_frame = evalita_preds_to_frames.get(predicate)
+                    has_relevant_frame = z_shot_frame in frames_of_interest or evalita_frame in frames_of_interest
+
+                    if z_shot_frame is not None:
+                        num_z_shot_all_predictions += 1
+                        if z_shot_frame in frames_of_interest:
+                            num_z_shot_foi_predictions += 1
+
+                    if evalita_frame is not None:
+                        num_evalita_all_predictions += 1
+                        if evalita_frame in frames_of_interest:
+                            num_evalita_foi_predictions += 1
+
+                    num_all_predictions += 1
+                    if has_relevant_frame:
+                        num_foi_predictions += 1
+
+                    if z_shot_frame != evalita_frame:
+                        diff = {
+                            "ev_id": ev_id,
+                            "doc_id": doc_id,
+                            "sent_idx": sent_idx,
+                            "sentence": " ".join(z_shot_sent["tokens"]),
+                            "predicate": predicate,
+                            "predicted_frame_0shot": z_shot_frame or "_",
+                            "predicted_frame_evalita": evalita_frame or "_"
+                        }
+                        all_differences.append(diff)
+                        if has_relevant_frame:
+                            foi_differences.append(diff)
+
+                print()
+
+    print()
+
+    print(f"num_z_shot_all_predictions = {num_z_shot_all_predictions}")
+    print(f"num_z_shot_foi_predictions = {num_z_shot_foi_predictions}")
+    print(f"num_evalita_all_predictions = {num_evalita_all_predictions}")
+    print(f"num_evalita_foi_predictions = {num_evalita_foi_predictions}")
+
+    print(
+        f"all_differences: {len(all_differences)}/{num_all_predictions}={len(all_differences)/num_all_predictions}")
+    print(
+        f"foi_differences: {len(foi_differences)}/{num_foi_predictions}={len(foi_differences) / num_foi_predictions}")
+
+    # all_diffs_df = pd.DataFrame(all_differences)
+    # foi_diffs_df = pd.DataFrame(foi_differences)
+
+    # all_diffs_df.to_csv("output/femicides/compare_lome_models/all_differences.csv")
+    # foi_diffs_df.to_csv("output/femicides/compare_lome_models/foi_differences.csv")
+
+    # annotator_sheet, experiment_sheet = make_evaluation_sample(foi_diffs_df)
+    # annotator_sheet.to_csv("output/femicides/compare_lome_models/annotator_sheet.csv")
+    # experiment_sheet.to_csv("output/femicides/compare_lome_models/experiment_sheet.csv")
+
+
+def analyze_annotations():
+    ann_df = pd.read_excel("resources/sara_lome_annotations.xlsx", index_col=0)
+    exp_df = pd.read_csv(
+        f"output/femicides/compare_lome_models/{OUT_FOLDER}/experiment_sheet.csv", index_col=0)
+    ann_df_ = ann_df.join(exp_df[["model_1", "model_2"]])
+    ann_df_proc = ann_df_.apply(combine_labels, axis=1)
+    print(ann_df_proc.head())
+    ann_df_proc.to_csv(
+        f"output/femicides/compare_lome_models/{OUT_FOLDER}/annotator_sheet_processed.csv")
+
+
+def combine_labels(row: pd.Series) -> pd.Series:
+
+    model_1 = row["model_1"].split("_")[-1]
+    model_2 = row["model_2"].split("_")[-1]
+
+    if row["answer::1_is_best"] == "X":
+        answer = f"{model_1}_is_best"
+    elif row["answer::2_is_best"] == "X":
+        answer = f"{model_2}_is_best"
+    elif row["answer::both_are_good"] == "X":
+        answer = "both_are_good"
+    elif row["answer::both_are_bad"] == "X":
+        answer = "both_are_bad"
+    elif row["answer::missing_frame"] == "X":
+        answer = "missing_frame"
+    else:
+        raise ValueError(f"Missing annotation in row {row}")
+
+    row_ = row.drop([k for k in row.keys() if k.startswith("answer::")])
+    return row_.append(pd.Series({"answer": answer}))
+
+
+def prep_svm_challenge():
+    annotated_df = pd.read_csv(
+        "output/femicides/compare_lome_models/0shot__vs__evalita_plus_fn/annotator_sheet_processed.csv", index_col=0)
+    
+    evalita_train_data = []
+    with open("../stupid-svm-frameid/data/evalita_jsonl/evalita_train.jsonl", encoding="utf-8") as f_in:
+        for line in f_in:
+            evalita_train_data.append(json.loads(line))
+    # evalita_frame_labels = {annotation["label"] for sentence in evalita_train_data for annotation in sentence["annotations"]}
+    evalita_frame_labels = defaultdict(int)
+    for sentence in evalita_train_data:
+        for annotation in sentence["annotations"]:
+            evalita_frame_labels[annotation["label"]] += 1
+    evalita_train_counts = pd.DataFrame(evalita_frame_labels.items(), columns=["label", "count"]).sort_values(by="count")
+    evalita_train_counts.to_csv("output/femicides/compare_lome_models/evalita_trainset_counts.csv")
+
+    print("Evalita frame labels:", sorted(evalita_frame_labels.keys()))
+
+    out = []
+    zshot_score = 0
+    evalita_score = 0
+
+    for _, row in annotated_df.iterrows():
+        answer = row["answer"]
+        if answer not in ["0shot_is_best", "evalita_is_best", "both_are_good"]:
+            continue
+        
+        tokens = row["sentence"].split()
+        predicate = row["predicate"].split("_")[0]  # to keep things simple, only look at first token of predicate
+        predicate_idx = [i for i, tok in enumerate(tokens) if tok == predicate][0]
+        
+        if answer == "0shot_is_best":
+            if row["model_1"] == "predicted_frame_0shot": 
+                zshot_label = label = row["prediction_1"]
+                evalita_label = row["prediction_2"]
+            else:
+                zshot_label = label = row["prediction_2"]
+                evalita_label = row["prediction_1"]
+        elif answer == "evalita_is_best":
+            if row["model_1"] == "predicted_frame_evalita": 
+                evalita_label = label = row["prediction_1"]
+                zshot_label = row["prediction_2"]
+            else:
+                evalita_label = label = row["prediction_2"]
+                zshot_label = row["prediction_1"]
+        else:
+            label = row["prediction_1"]
+            if row["model_1"] == "predicted_frame_evalita": 
+                evalita_label = row["prediction_1"]
+                zshot_label = row["prediction_2"]
+            else:
+                evalita_label = row["prediction_2"]
+                zshot_label = row["prediction_1"]
+
+        if label not in evalita_frame_labels:
+            print("\tskipping gold frame label not present in EVALITA: ", label)
+            continue
+
+        if zshot_label == label:
+            zshot_score += 1
+        if evalita_label == label:
+            evalita_score += 1
+
+        out.append({"tokens": tokens, "annotations": [{"label": label, "span": [predicate_idx, predicate_idx], "lu": None, "children": []}]})
+
+    print(f"Found {len(out)} relevant annotations")
+    print("0-shot score: ", zshot_score / len(out))
+    print("evalita score: ", evalita_score / len(out))
+
+
+    with open("output/femicides/compare_lome_models/svm_challenge.jsonl", "w", encoding="utf-8") as f_out:
+        for line in out:
+            f_out.write(json.dumps(line) + os.linesep)
+        f_out.write(os.linesep)
+
+
+
+if __name__ == '__main__':
+    action = sys.argv[1]
+    assert action in ["make", "analyze", "prep_svm_challenge"]
+
+    if action == "make":
+        make_annotation_experiment()
+    elif action == "analyze":
+        analyze_annotations()
+    else:
+        prep_svm_challenge()

sociofillmore/femicides/evalita_err_analysis.py

@@ -0,0 +1,182 @@
+from itertools import product
+
+import pandas as pd
+import numpy as np
+from scipy.spatial.distance import cosine
+
+from nltk.corpus import framenet as fn
+
+from sociofillmore.common.analyze_text import read_frames_of_interest
+
+
+COSINE_THRESH = [0.1, 0.2, 0.3, 0.4, 0.5]
+
+
+PREDICTION_FILES = {
+    "evalita-dev": {
+        "stupid-svm": "../stupid-svm-frameid/evalita_predictions.csv",
+        "lome-en": "misc/frame_prediction_output_lome-en_dev.csv",
+        "lome-it": "misc/frame_prediction_output_lome-it-best_dev.csv",
+    },
+    "evalita-test": {
+        "stupid-svm": "../stupid-svm-frameid/evalita_predictions_test.csv",
+        "lome-en": "misc/frame_prediction_output_lome-en_test.csv",
+        "lome-it": "misc/frame_prediction_output_lome-it-best_test.csv",
+    },
+    "rai_femicides": {
+        "stupid-svm": "../stupid-svm-frameid/rai_predictions.csv",
+        "lome-en": "misc/frame_prediction_output_lome-en_rai.csv",
+        "lome-it": "misc/frame_prediction_output_lome-it-best_rai.csv",
+    },
+}
+
+
+def load_embeddings(embedding_file):
+    frame_vocab = []
+    word_vocab = []
+    vectors = []
+
+    with open(embedding_file, encoding="utf-8") as f:
+        for line in f:
+            columns = line.split()
+            frame = columns[0]
+            words = tuple(columns[1].split("+"))
+            vector = np.array([float(i) for i in columns[2:]])
+
+            frame_vocab.append(frame)
+            word_vocab.append(words)
+            vectors.append(vector)
+
+    frames_to_idxs = {}
+    for i, frame in enumerate(frame_vocab):
+        frames_to_idxs[frame] = i
+
+    return np.array(vectors, dtype=np.float64), frames_to_idxs
+
+
+def femicide_frame_distances(embeddings, frame_to_idx):
+    femicide_frames = read_frames_of_interest("femicides/rai")
+    print("Cosines: ")
+    for fr1, fr2 in product(femicide_frames, femicide_frames):
+        dist = cosine(embeddings[frame_to_idx[fr1]], embeddings[frame_to_idx[fr2]])
+        print(f"\t{fr1}-{fr2}: {dist:.4f}")
+
+
+def embedding_scores(predictions, embeddings, frame_to_idx):
+    correct = 0
+    close_calls = {threshold: 0 for threshold in COSINE_THRESH}
+    total_dist = 0.0
+
+    for _, row in predictions.iterrows():
+        predicted = row["frame_pred"]
+        gold = row["frame_gold"]
+        dist = cosine(
+            embeddings[frame_to_idx[predicted]], embeddings[frame_to_idx[gold]]
+        )
+        if predicted == gold:
+            correct += 1
+        else:
+            for threshold in COSINE_THRESH:
+                if dist < threshold:
+                    close_calls[threshold] += 1
+        total_dist += dist
+
+    print("#correct: ", correct / len(predictions))
+    print("#close calls: ")
+    for threshold in COSINE_THRESH:
+        print("\t", threshold, (close_calls[threshold]) / len(predictions))
+    print("#correct or close: ")
+    for threshold in COSINE_THRESH:
+        print("\t", threshold, (correct + close_calls[threshold]) / len(predictions))
+    print("avg cosine dist: ", total_dist / len(predictions))
+
+
+def generalization_exp(predictions, evalita_train_counts, fn_frames, femicide_frames):
+
+    all_frames = predictions
+    ifn_frames = predictions[
+        predictions["frame_gold"].isin(evalita_train_counts["label"])
+    ]
+    bfn_frames = predictions[predictions["frame_gold"].isin(fn_frames)]
+    rai_frames = predictions[predictions["frame_gold"].isin(femicide_frames)]
+
+
+    print("LEN (ALL/IFN/BFN/RAI:)")
+    print(
+        "\t".join(
+            [
+                str(len(preds))
+                for preds in [all_frames, ifn_frames, bfn_frames, rai_frames]
+            ]
+        )
+    )
+
+    print("ACC (ALL/IFN/BFN/RAI:)")
+    print(
+        "\t".join(
+            [
+                str(len(preds[preds["frame_gold"] == preds["frame_pred"]]) / len(preds))
+                for preds in [all_frames, ifn_frames, bfn_frames, rai_frames]
+            ]
+        )
+    )
+
+
+def main():
+
+    evalita_train_counts = pd.read_csv(
+        "output/femicides/compare_lome_models/evalita_trainset_counts.csv"
+    )
+
+    fn_frames = {fr.name for fr in fn.frames()}
+    femicide_frames = read_frames_of_interest("femicides/rai")
+    evalita_train_counts = pd.read_csv(
+        "output/femicides/compare_lome_models/evalita_trainset_counts.csv"
+    )
+
+    for dataset in PREDICTION_FILES:
+        print(f"==={dataset}===")
+        for model, predictions_file in PREDICTION_FILES[dataset].items():
+
+            print(f"---{model}---")
+
+            predictions = pd.read_csv(predictions_file, index_col=0)
+            print("Total predictions:", len(predictions))
+
+            # predictions_with_fn_frames = predictions[
+            #     predictions["frame_gold"].isin(fn_frames)
+            #     & predictions["frame_pred"].isin(fn_frames)
+            # ]
+            # print("Predictions with FN frames: ", len(predictions_with_fn_frames))
+
+            # errors = predictions[predictions["frame_gold"] != predictions["frame_pred"]]
+            # print("Total errors: ", len(errors))
+
+            # errors_with_fn_frames = errors[
+            #     errors["frame_gold"].isin(fn_frames) & errors["frame_pred"].isin(fn_frames)
+            # ]
+            # print("Errors with FN frames: ", len(errors_with_fn_frames))
+
+            # print("Loading embeddings...")
+            # embeddings, frame_to_idx = load_embeddings(
+            #     "../bert-for-framenet/data/embeddings/bag_of_lu_embeddings.txt"
+            # )
+            # # femicide_frame_distances(embeddings, frame_to_idx)
+            # embedding_scores(predictions_with_fn_frames, embeddings, frame_to_idx)
+
+            if dataset == "rai_femicides":
+                predictions = predictions[predictions["frame_gold"].isin(femicide_frames)]
+
+
+            femicide_frames = read_frames_of_interest("femicides/rai")
+            generalization_exp(
+                predictions, evalita_train_counts, fn_frames, femicide_frames
+            )
+
+
+            print()
+        print()
+
+
+if __name__ == "__main__":
+    main()

sociofillmore/femicides/extract_texts.py

@@ -0,0 +1,15 @@
+import pandas as pd
+
+
+def extract_texts():
+    df = pd.read_excel("data/femicides/rai/EventiFemminicidio_from2015to2017_fonti.xlsx")
+    print(df)
+    for _, row in df.iterrows():
+        source_id = row["ID"]
+        text = f"{row['title']}\n\n{row['text']}"
+        with open(f"output/femicides/extract_text/source_{source_id}.txt", "w", encoding="utf-8") as f:
+            f.write(text)
+
+
+if __name__ == '__main__':
+    extract_texts()

sociofillmore/femicides/split_data.py

@@ -0,0 +1,235 @@
+"""
+Split the dataset into a "DEV10" portion (10% of events) for initial experimentation; and "MAIN", the rest of the
+dataset, to be used later
+"""
+import os
+import json
+import random
+import argparse
+from collections import defaultdict
+from typing import List, Tuple, Dict, Any
+
+import pandas as pd
+import nltk
+
+random.seed(1996)
+
+
+def split_rai_femicides():
+    # process the excel file
+    print("Processing excel file...")
+    femicide_events = pd.read_excel("data/femicides/rai/EventiFemminicidio_from2015to2017.edited_colnames.xlsx",
+                                    sheet_name="dati", header=0)
+    event_ids, victim_to_event_id, event_id_to_victims, victim_duplicate_counts = read_events(femicide_events)
+
+    # save information about correspondences between victims and events
+    # (we will need this later to retrieve the correct texts for each event, because the XLSX with texts uses victim
+    # names as keys)
+    dicts_to_save = (
+        (victim_to_event_id, "victim_to_event_id"),
+        (event_id_to_victims, "event_id_to_victims"),
+        (victim_duplicate_counts, "victim_duplicate_counts")
+    )
+    write_dict_to_json(dicts_to_save)
+
+    # shuffle and split
+    print("Shuffling and splitting...")
+    shuffled_event_ids = list(event_ids)
+    random.shuffle(shuffled_event_ids)
+    dev10_idx = shuffled_event_ids[:78]
+    main_idx = shuffled_event_ids[78:]
+    dev10_df, main_df = create_split_df(dev10_idx, femicide_events)
+
+    # write split dataframes
+    for df, df_name in ((dev10_df, "dev10"), (main_df, "main")):
+        df.to_csv(f"output/femicides/split_data/rai/split_{df_name}.events.csv")
+        df.to_excel(f"output/femicides/split_data/rai/split_{df_name}.events.xlsx")
+
+    # write filtered victim data
+    dev10_victims = {e: victims for e, victims in event_id_to_victims.items() if e in dev10_idx}
+    main_victims = {e: victims for e, victims in event_id_to_victims.items() if e in main_idx}
+    filtered_dicts_to_save = (
+        (dev10_victims, "event_id_to_victims.dev10"),
+        (main_victims, "event_id_to_victims.main"),
+    )
+    write_dict_to_json(filtered_dicts_to_save)
+
+    # retrieve texts for filtered data
+    print("Filtering & writing texts...")
+    texts_df = pd.read_excel("data/femicides/rai/EventiFemminicidio_from2015to2017_fonti.xlsx")
+    filter_texts("dev10", texts_df, dev10_victims, victim_duplicate_counts)
+    filter_texts("main", texts_df, main_victims, victim_duplicate_counts)
+
+
+def split_olv_femicides():
+    texts_df = pd.read_csv("data/femicides/olv/texts_scrape_match_scrape_2021-10-28.csv")
+    events_df = pd.read_csv("data/femicides/olv/events_scrape_match_scrape_2021-10-28.csv")
+
+    event_ids = events_df["event:id"].tolist()
+    random.shuffle(event_ids)
+
+    num_dev_events = round(len(event_ids) * 0.10)
+    dev10_ids = event_ids[:num_dev_events]
+    dev10_df, main_df = create_split_df(dev10_ids, events_df)
+
+    # split texts
+    dev10_texts_df = texts_df[texts_df["event_id"].isin(dev10_ids)]
+    main_texts_df = texts_df[~texts_df["event_id"].isin(dev10_ids)]
+
+    # write to files
+    for events_df, texts_df, split_name in ((dev10_df, dev10_texts_df, "dev10"), (main_df, main_texts_df, "main")):
+        events_df.to_csv(f"output/femicides/split_data/olv/split_{split_name}.events.csv")
+        texts_df.to_csv(f"output/femicides/split_data/olv/split_{split_name}.texts.csv")
+        events_df.to_excel(f"output/femicides/split_data/olv/split_{split_name}.events.xlsx")
+        texts_df.to_excel(f"output/femicides/split_data/olv/split_{split_name}.texts.xlsx")
+
+        for _, row in texts_df.iterrows():
+            event_id = row["event_id"]
+            text_id = row["text_id"]
+            event_dir = f"output/femicides/split_data/olv/split_{split_name}_texts_by_event/{event_id}/"
+            os.makedirs(event_dir, exist_ok=True)
+            with open(event_dir + f"/{text_id}.txt", "w", encoding="utf-8") as f_by_event:
+                title = row["title"] if not pd.isna(row["title"]) else ""
+                for line in nltk.sent_tokenize(title, language="italian"):
+                    f_by_event.write(line + os.linesep)
+                f_by_event.write(os.linesep)
+                fulltext = row["fulltext"] if not pd.isna(row["fulltext"]) else ""
+                if not fulltext:
+                    print(f"WARNING: empty fulltext in text_id={text_id}")
+                for line in nltk.sent_tokenize(fulltext, language="italian"):
+                    line = line.strip()
+                    if not line:
+                        continue
+                    f_by_event.write(line + os.linesep)
+
+
+def write_dict_to_json(filtered_dicts_to_save):
+    for dict_data, dict_name in filtered_dicts_to_save:
+        with open(f"output/femicides/split_data/rai/{dict_name}.json", "w", encoding="utf-8") as f:
+            json.dump(dict_data, f, indent=4, sort_keys=True)
+
+
+def create_split_df(dev10: List[int], femicide_events: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
+    dev10_rows = []
+    main_rows = []
+    for idx, row in femicide_events.iterrows():
+        event_id = row["event:id"]
+        if pd.isna(event_id):
+            continue
+        event_id = int(event_id)
+        if event_id in dev10:
+            dev10_rows.append(row)
+        else:
+            main_rows.append(row)
+    dev10_df = pd.DataFrame(dev10_rows)
+    main_df = pd.DataFrame(main_rows)
+    return dev10_df, main_df
+
+
+def read_events(events_df):
+    event_ids: List[int] = []
+    victim_to_event_id: Dict[str, int] = {}
+    event_id_to_victims: Dict[int, List[Tuple[str, int]]] = defaultdict(list)
+    victim_duplicate_counts: Dict[str, int] = defaultdict(int)
+
+    for idx, row in events_df.iterrows():
+        event_id = row["event:id"]
+        if pd.isna(event_id):
+            continue
+        event_id = int(event_id)
+
+        # unspecified name --> "UNKNOWN_X"
+        victim = row["victim:name"]
+        if victim == "non rilevato" or pd.isna(victim):
+            victim = f"UNKNOWN_{event_id}"
+
+        # disambiguate victims with duplicate names
+        victim_duplicate_counts[victim] += 1
+        duplicate_id = victim_duplicate_counts[victim]
+
+        event_ids.append(event_id)
+        victim_to_event_id[f"{victim}/{duplicate_id}"] = event_id
+        event_id_to_victims[event_id].append((victim, duplicate_id))
+    return event_ids, victim_to_event_id, event_id_to_victims, victim_duplicate_counts
+
+
+def filter_texts(split_name: str,
+                 texts_df: pd.DataFrame,
+                 event_idx_to_victims: Dict[int, List[Tuple[str, int]]],
+                 victim_duplicate_counts: Dict[str, int]):
+
+    print(f"\tfilter_texts: filtering split {split_name}")
+
+    # first filter victims
+    victim_to_event_idx = {}
+    for e_id in event_idx_to_victims:
+        for victim in event_idx_to_victims[e_id]:
+            victim_name, victim_dup_id = victim
+            if victim_duplicate_counts[victim_name] > 1:
+                print(f"\tfilter_texts: removing ambiguous victim name '{victim_name}'")
+                continue
+            victim_to_event_idx[victim_name] = e_id
+
+    meta_rows: List[Dict[str, Any]] = []
+    with open(f"output/femicides/split_data/rai/split_{split_name}.texts.text.txt", "w", encoding="utf-8") as f_txt, \
+            open(f"output/femicides/split_data/rai/split_{split_name}.texts.ids.txt", "w", encoding="utf-8") as f_id:
+        for _, row in texts_df.iterrows():
+            text_victim = row["vittima"].strip()
+            if text_victim in victim_to_event_idx:
+                e_id = victim_to_event_idx[text_victim]
+                text_id = int(row["ID"])
+                url = row["link"]
+                pubdate = row["pubdate"]
+                provider = row["provider"]
+                title = row["title"]
+
+                meta_rows.append({
+                    "event_id": e_id,
+                    "text_id": text_id,
+                    "url": url,
+                    "pubdate": pubdate,
+                    "provider": provider,
+                    "title": title
+                })
+
+                # body_text_lines = row["text"].split("\n")
+                body_text_lines = nltk.sent_tokenize(row["text"], language="italian")
+                title_lines = nltk.sent_tokenize(title, language="italian")
+
+                # f_txt.write(title.strip() + os.linesep)
+                # f_id.write(f"event {e_id}\ttext {text_id}\ttitle" + os.linesep)
+                for line in title_lines:
+                    f_txt.write(line + os.linesep)
+                    f_id.write(f"event {e_id}\ttext {text_id}\ttitle" + os.linesep)
+
+                event_dir = f"output/femicides/split_data/rai/split_{split_name}_texts_by_event/{e_id}/"
+                os.makedirs(event_dir, exist_ok=True)
+                with open(event_dir + f"/{text_id}.txt", "w", encoding="utf-8") as f_by_event:
+                    # f_by_event.write(title.strip() + os.linesep)
+                    for line in title_lines:
+                        f_by_event.write(line + os.linesep)
+                    f_by_event.write(os.linesep)
+                    for line in body_text_lines:
+                        line = line.strip()
+                        if not line:
+                            continue
+                        f_txt.write(line + os.linesep)
+                        f_by_event.write(line + os.linesep)
+                        f_id.write(f"event {e_id}\ttext {text_id}\tbody" + os.linesep)
+
+    meta_df = pd.DataFrame(meta_rows)
+    meta_df.to_csv(f"output/femicides/split_data/rai/split_{split_name}.texts.meta.csv")
+    meta_df.to_excel(f"output/femicides/split_data/rai/split_{split_name}.texts.meta.xlsx")
+    print()
+
+
+if __name__ == '__main__':
+
+    ap = argparse.ArgumentParser()
+    ap.add_argument("dataset", choices=["rai", "olv"])
+    args = ap.parse_args()
+
+    if args.dataset == "rai":
+        split_rai_femicides()
+    else:
+        split_olv_femicides()

sociofillmore/migration/cda_classify.py

@@ -0,0 +1,338 @@
+"""
+Learn to classify the manually annotated CDA attributes (frames, 'riferimento', orientation)
+"""
+
+import sys
+
+import torch
+
+from allennlp.data.vocabulary import Vocabulary
+from allennlp.data import DatasetReader, TokenIndexer, Instance, Token
+from allennlp.data.fields import TextField, LabelField
+from allennlp.data.token_indexers.pretrained_transformer_indexer import (
+    PretrainedTransformerIndexer,
+)
+from allennlp.data.tokenizers.pretrained_transformer_tokenizer import (
+    PretrainedTransformerTokenizer,
+)
+from allennlp.models import BasicClassifier
+from allennlp.modules.text_field_embedders.basic_text_field_embedder import (
+    BasicTextFieldEmbedder,
+)
+from allennlp.modules.token_embedders.pretrained_transformer_embedder import (
+    PretrainedTransformerEmbedder,
+)
+from allennlp.modules.seq2vec_encoders.bert_pooler import BertPooler
+from allennlp.training.checkpointer import Checkpointer
+from allennlp.training.gradient_descent_trainer import GradientDescentTrainer
+from allennlp.data.data_loaders.simple_data_loader import SimpleDataLoader
+from allennlp.training.optimizers import AdamOptimizer
+from allennlp.predictors.text_classifier import TextClassifierPredictor
+
+from sklearn.svm import SVC
+from sklearn.feature_extraction.text import CountVectorizer
+from sklearn.metrics import precision_recall_fscore_support
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.dummy import DummyClassifier
+
+import pandas as pd
+import numpy as np
+import spacy
+
+import json
+import os
+from typing import Dict, Iterable
+
+
+class MigrationReader(DatasetReader):
+    def __init__(self, token_indexers, tokenizer):
+        self.token_indexers = token_indexers
+        self.tokenizer = tokenizer
+
+    def text_to_instance(self, sentence, label=None) -> Instance:
+        text_field = TextField(self.tokenizer.tokenize(sentence), self.token_indexers)
+        fields = {"tokens": text_field}
+        if label is not None:
+            label_field = LabelField(label)
+            fields["label"] = label_field
+        return Instance(fields)
+
+
+    def read_instances(
+        self, text: pd.Series, labels: pd.Series
+    ) -> Iterable[Instance]:
+        for sentence, label in zip(text, labels):
+            instance = self.text_to_instance(sentence, label)
+            yield instance
+
+
+def train(attrib, use_gpu=False):
+    assert attrib in ["cda_frame", "riferimento", "orientation", "fake"]
+
+    # load data
+    print("Loading data...")
+    x_train, y_train, x_dev, y_dev = load_data(attrib)
+    print(f"\t\ttrain size: {len(x_train)}")
+    print(f"\t\tdev size: {len(x_dev)}")
+
+    # try different setups
+    print("Running training setups...")
+    scores = []
+    setups = [
+        # defaults: remove_punct=True, lowercase=True, lemmatize=False, remove_stop=False
+        # ({}, {}, {"type": "svm", "options": {"kernel": "linear", "C": 1.0}}),
+        (
+            {},
+            {},
+            {
+                "type": "bert",
+                "options": {"transformer": "Musixmatch/umberto-commoncrawl-cased-v1"},
+            },
+        ),
+        # ({"lemmatize": True, "remove_stop": True}, {}, {"type": "svm", "options": {"kernel": "linear", "C": 0.8}}),
+        # ({"lemmatize": True, "remove_stop": True}, {"embed": False}, {"type": "svm", "options": {"kernel": "linear", "C": 0.8}}),
+        # ({"lemmatize": True, "remove_stop": True}, {"embed": False}, {"type": "dummy", "options": {}}),
+        # ({"lemmatize": True, "remove_stop": True}, {"embed": False}, {"type": "tree", "options": {}}),
+        # ({"lemmatize": True, "remove_stop": True}, {}, SVC(kernel='linear')),
+        # ({"lemmatize": True, "remove_stop": True}, {"min_freq": 5}, SVC(kernel='linear')),
+        # ({"lemmatize": True, "remove_stop": True}, {"min_freq": 5, "max_freq": .70}, SVC(kernel='linear')),
+        # ({"lemmatize": True, "remove_stop": True}, {}, SVC(kernel='linear', C=0.6)),
+        # ({"lemmatize": True, "remove_stop": True}, {}, SVC(kernel='linear', C=0.7)),
+        # ({"lemmatize": True, "remove_stop": True}, {}, SVC(kernel='linear', C=0.8)),
+        # ({"lemmatize": True, "remove_stop": True}, {"ngram_range": (1,2)}, SVC(kernel='linear', C=0.8)),
+        # ({"lemmatize": True, "remove_stop": True}, {}, SVC(kernel="rbf")),
+    ]
+
+    nlp = spacy.load("it_core_news_md")
+
+    for s_idx, (text_options, vect_options, model_info) in enumerate(setups):
+
+        if model_info["type"] == "bert":
+            print("\t\tPreparing BERT model...")
+
+            # cuda_device = 0 if torch.cuda.is_available() else -1
+            cuda_device = None if use_gpu and torch.cuda.is_available() else -1
+
+            transformer = model_info["options"]["transformer"]
+            token_indexers = {"tokens": PretrainedTransformerIndexer(transformer)}
+            tokenizer = PretrainedTransformerTokenizer(transformer)
+
+            reader = MigrationReader(token_indexers, tokenizer)
+            train_instances = list(
+                reader.read_instances(x_train, y_train)
+            )
+            dev_instances = list(
+                reader.read_instances(x_dev, y_dev)
+            )
+            vocab = Vocabulary.from_instances(train_instances + dev_instances)
+            print(vocab.get_vocab_size("tags"))
+
+            embedder = BasicTextFieldEmbedder(
+                {"tokens": PretrainedTransformerEmbedder(transformer)}
+            )
+            seq2vec = BertPooler(transformer)
+            model = BasicClassifier(vocab, embedder, seq2vec, namespace="tags")        
+            if use_gpu:
+                model = model.cuda(cuda_device)
+
+            checkpoint_dir = f"/scratch/p289731/cda_classify/model_{attrib}/checkpoints/"
+            serialization_dir = f"/scratch/p289731/cda_classify/model_{attrib}/serialize/"
+            os.makedirs(checkpoint_dir)
+            os.makedirs(serialization_dir)
+            checkpointer = Checkpointer(checkpoint_dir)
+            optimizer = AdamOptimizer(
+                [(n, p) for n, p in model.named_parameters() if p.requires_grad],
+                lr=1e-6
+            )
+            train_loader = SimpleDataLoader(train_instances, batch_size=8, shuffle=True)
+            dev_loader = SimpleDataLoader(dev_instances, batch_size=8, shuffle=False)
+            train_loader.index_with(vocab)
+            dev_loader.index_with(vocab)
+
+            print("\t\tTraining BERT model")
+            trainer = GradientDescentTrainer(
+                model,
+                optimizer,
+                train_loader,
+                validation_data_loader=dev_loader,
+                patience=32,
+                checkpointer=checkpointer,
+                cuda_device=cuda_device,
+                serialization_dir=serialization_dir
+            )
+            trainer.train()
+
+            print("\t\tProducing predictions...")
+            predictor = TextClassifierPredictor(model, reader)
+            predictions = [predictor.predict(sentence) for sentence in x_dev] 
+            y_dev_pred = [p["label"] for p in predictions]
+            class_labels = list(vocab.get_token_to_index_vocabulary("labels").keys())
+
+        elif model_info["type"] in ["svm", "tree", "dummy"]:
+            # extract features
+            print("\t\tExtracting features...")
+            x_train_fts, vectorizer = extract_features(
+                x_train, nlp, text_options, **vect_options
+            )
+            x_dev_fts, _ = extract_features(
+                x_dev, nlp, text_options, **vect_options, vectorizer=vectorizer
+            )
+
+            if not vect_options["embed"]:
+                print(f"\t\t\tnum features: {len(vectorizer.vocabulary_)}")
+            else:
+                assert model_info["type"] != "tree", "Decision tree does not support embedding input"
+
+            print("\t\tTraining the model...")
+            if model_info["type"] == "svm":
+                model = SVC(**model_info["options"])
+            elif model_info["type"] == "tree":
+                model = DecisionTreeClassifier()
+            else:
+                model = DummyClassifier()
+            model.fit(x_train_fts, y_train)
+
+            # evaluate on dev
+            print("\t\tValidating the model...")
+            y_dev_pred = model.predict(x_dev_fts)
+            class_labels = model.classes_
+
+        p_micro, r_micro, f_micro, _ = precision_recall_fscore_support(
+            y_dev, y_dev_pred, average="micro"
+        )
+        p_classes, r_classes, f_classes, _ = precision_recall_fscore_support(
+            y_dev, y_dev_pred, average=None, labels=class_labels, zero_division=0
+        )
+        print(
+            f"\t\t\tOverall scores (micro-averaged):\tP={p_micro}\tR={r_micro}\tF={f_micro}"
+        )
+
+        scores.append(
+            {
+                "micro": {"p": p_micro, "r": r_micro, "f": f_micro},
+                "classes": {
+                    "p": list(zip(class_labels, p_classes)),
+                    "r": list(zip(class_labels, r_classes)),
+                    "f": list(zip(class_labels, f_classes)),
+                },
+            }
+        )
+
+        prediction_df = pd.DataFrame(
+            zip(x_dev, y_dev, y_dev_pred), columns=["headline", "gold", "prediction"]
+        )
+        prediction_df.to_csv(
+            f"output/migration/cda_classify/predictions_{attrib}_{s_idx:02}.csv"
+        )
+
+    with open(
+        f"output/migration/cda_classify/scores_{attrib}.json", "w", encoding="utf-8"
+    ) as f_scores:
+        json.dump(scores, f_scores, indent=4)
+
+
+def load_data(attrib):
+    train_data = pd.read_csv("output/migration/preprocess/annotations_train.csv")
+    dev_data = pd.read_csv("output/migration/preprocess/annotations_dev.csv")
+
+    x_train = train_data["Titolo"]
+    x_dev = dev_data["Titolo"]
+
+    if attrib == "cda_frame":
+        y_train = train_data["frame"]
+        y_dev = dev_data["frame"]
+    elif attrib == "riferimento":
+        y_train = train_data["riferimento"]
+        y_dev = dev_data["riferimento"]
+    elif attrib == "orientation":
+        y_train = train_data["orientation"]
+        y_dev = dev_data["orientation"]
+
+    # fake task to test setup
+    else:
+        y_train = pd.Series(["true" if "rifugiato" in exa else "false" for exa in x_train])
+        y_dev = pd.Series(["true" if "rifugiato" in exa else "false" for exa in x_dev])
+
+    return x_train, y_train, x_dev, y_dev
+
+
+def extract_features(
+    headlines,
+    nlp,
+    text_options,
+    embed=False,
+    min_freq=1,
+    max_freq=1.0,
+    ngram_range=(1, 1),
+    vectorizer=None,
+):
+
+    if embed:
+        vectorized = np.array(
+            [vec for vec in process_text(headlines, nlp, embed=True, **text_options)]
+        )
+    else:
+        tokenized = [
+            " ".join(sent) for sent in process_text(headlines, nlp, **text_options)
+        ]
+        if vectorizer is None:
+            vectorizer = CountVectorizer(
+                lowercase=False,
+                analyzer="word",
+                min_df=min_freq,
+                max_df=max_freq,
+                ngram_range=ngram_range,
+            )
+            vectorized = vectorizer.fit_transform(tokenized)
+        else:
+            vectorized = vectorizer.transform(tokenized)
+    return vectorized, vectorizer
+
+
+def process_text(
+    headlines,
+    nlp,
+    embed=False,
+    remove_punct=True,
+    lowercase=True,
+    lemmatize=False,
+    remove_stop=False,
+):
+    for sent in headlines:
+        doc = nlp(sent)
+        tokens = (
+            t
+            for t in doc
+            if (not remove_stop or not t.is_stop)
+            and (not remove_punct or t.pos_ not in ["PUNCT", "SYM", "X"])
+        )
+        if embed:
+            if lemmatize:
+                tokens = (t.vocab[t.lemma].vector for t in tokens)
+            else:
+                tokens = (t.vector for t in tokens if t.has_vector)
+        else:
+            if lemmatize:
+                tokens = (t.lemma_ for t in tokens)
+            else:
+                tokens = (t.text for t in tokens)
+
+            if lowercase:
+                tokens = (t.lower() for t in tokens)
+
+        if embed:
+            token_arr = np.array([t for t in tokens])
+            if len(token_arr) == 0:
+                yield np.random.rand(300)
+            else:
+                yield np.mean(token_arr, axis=0)
+        else:
+            yield list(tokens)
+
+
+if __name__ == "__main__":
+    use_gpu = True if sys.argv[1] == "gpu" else False
+    # train(attrib="fake", use_gpu=use_gpu)
+    train(attrib="cda_frame", use_gpu=use_gpu)
+    # train(attrib="riferimento")
+    # train(attrib="orientation")

sociofillmore/migration/cda_classify_.py

@@ -0,0 +1,140 @@
+"""
+Learn to classify the manually annotated CDA attributes (frames, 'riferimento', orientation)
+"""
+
+GLOVE_MODEL = "/net/aistaff/gminnema/thesis_data/data/glove-it/glove_WIKI"
+
+
+from sklearn.svm import SVC
+from sklearn.feature_extraction.text import CountVectorizer
+from sklearn.metrics import precision_recall_fscore_support
+import gensim
+import pandas as pd
+import spacy
+
+import json
+
+
+def train(attrib):
+    assert attrib in ["cda_frame", "riferimento", "orientation"]
+
+    # load data
+    print("Loading data...")
+    x_train, y_train, x_dev, y_dev = load_data(attrib)
+    print(f"\t\ttrain size: {len(x_train)}")
+    print(f"\t\tdev size: {len(x_dev)}")
+
+    # try different setups
+    print("Running training setups...")
+    scores = []
+    setups = [
+        # defaults: remove_punct=True, lowercase=True, lemmatize=False, remove_stop=False
+        # ({}, {}, SVC(kernel='linear')),
+        # ({"lemmatize": True, "remove_stop": True}, {}, SVC(kernel='linear')),
+        # ({"lemmatize": True, "remove_stop": True}, {"min_freq": 5}, SVC(kernel='linear')),
+        # ({"lemmatize": True, "remove_stop": True}, {"min_freq": 5, "max_freq": .70}, SVC(kernel='linear')),
+        # ({"lemmatize": True, "remove_stop": True}, {}, SVC(kernel='linear', C=0.6)),
+        # ({"lemmatize": True, "remove_stop": True}, {}, SVC(kernel='linear', C=0.7)),
+        # ({"lemmatize": True, "remove_stop": True}, {}, SVC(kernel='linear', C=0.8)),
+        ({"lemmatize": True, "remove_stop": True}, {"embed": "glove"}, SVC(kernel='linear', C=0.8)),
+        # ({"lemmatize": True, "remove_stop": True}, {}, SVC(kernel="rbf")),
+    ]
+
+
+    nlp = spacy.load("it_core_news_md")
+
+    for s_idx, (text_options, vect_options, model) in enumerate(setups):
+
+        print(f"\tSetup #{s_idx}")
+
+        # extract features
+        print("\t\tExtracting features...")
+        x_train_fts, vectorizer = extract_features(x_train, nlp, text_options, **vect_options)
+        x_dev_fts, _ = extract_features(x_dev, nlp, text_options, **vect_options, vectorizer=vectorizer)
+        print(f"\t\t\tnum features: {len(vectorizer.vocabulary_)}")
+
+        print("\t\tTraining the model...")
+        model.fit(x_train_fts, y_train)
+
+        # evaluate on dev
+        print("\t\tValidating the model...")
+        y_dev_pred = model.predict(x_dev_fts)
+        p_micro, r_micro, f_micro, _ = precision_recall_fscore_support(
+            y_dev, y_dev_pred, average="micro")
+        p_classes, r_classes, f_classes, _ = precision_recall_fscore_support(
+            y_dev, y_dev_pred, average=None, labels=model.classes_, zero_division=0)
+        print(
+            f"\t\t\tOverall scores (micro-averaged):\tP={p_micro}\tR={r_micro}\tF={f_micro}"
+        )
+
+        scores.append({
+            "micro": {
+                "p": p_micro,
+                "r": r_micro,
+                "f": f_micro
+            },
+            "classes": {
+                "p": list(zip(model.classes_, p_classes)),
+                "r": list(zip(model.classes_, r_classes)),
+                "f": list(zip(model.classes_, f_classes)),
+            }
+        })
+
+        prediction_df = pd.DataFrame(zip(x_dev, y_dev, y_dev_pred), columns=["headline", "gold", "prediction"])
+        prediction_df.to_csv(f"output/migration/cda_classify/predictions_{s_idx:02}.csv")
+
+
+    with open("output/migration/cda_classify/scores.json", "w", encoding="utf-8") as f_scores:
+        json.dump(scores, f_scores, indent=4)
+
+
+def load_data(attrib):
+    train_data = pd.read_csv(
+        "output/migration/preprocess/annotations_train.csv")
+    dev_data = pd.read_csv("output/migration/preprocess/annotations_dev.csv")
+
+    x_train = train_data["Titolo"]
+    x_dev = dev_data["Titolo"]
+
+    if attrib == "cda_frame":
+        y_train = train_data["frame"]
+        y_dev = dev_data["frame"]
+    elif attrib == "riferimento":
+        y_train = train_data["riferimento"]
+        y_dev = dev_data["riferimento"]
+    else:
+        x_train = train_data["orientation"]
+        y_dev = dev_data["orientation"]
+    return x_train, y_train, x_dev, y_dev
+
+
+def extract_features(headlines, nlp, text_options, min_freq=1, max_freq=1.0, embed=None, vectorizer=None):
+    tokenized = [" ".join(sent) for sent in tokenize(headlines, nlp, **text_options)]
+    if vectorizer is None:
+        if embed is None:
+            vectorizer = CountVectorizer(lowercase=False, analyzer="word", min_df=min_freq, max_df=max_freq)
+            vectorized = vectorizer.fit_transform(tokenized)
+        else:
+            vectorizer = gensim.models.
+    else:
+        vectorized = vectorizer.transform(tokenized)
+    return vectorized, vectorizer
+
+
+def tokenize(headlines, nlp, remove_punct=True, lowercase=True, lemmatize=False, remove_stop=False):
+    for sent in headlines:
+        doc = nlp(sent)
+        tokens = (
+            t.lemma_ if lemmatize else t.text
+            for t in doc
+            if (not remove_stop or not t.is_stop) and (not remove_punct or t.pos_ not in ["PUNCT", "SYM", "X"])
+        )
+        if lowercase:
+            tokens = [t.lower() for t in tokens]
+        else:
+            tokens = [t for t in tokens]
+        yield tokens
+
+
+if __name__ == '__main__':
+    train(attrib="cda_frame")

sociofillmore/migration/extract_political_ratings.py

@@ -0,0 +1,17 @@
+import pandas as pd
+
+
+def main():
+    corpus_df = pd.read_excel("data/migration/corpus_titoli_2013_2021_with_most_recent_years.xlsx")
+    ratings = corpus_df.groupby("Testata").first()[["Orientamento politico", "Religiosa?"]]
+    ratings.index.rename("provider", inplace=True)
+    ratings["political_stance"] = ratings["Orientamento politico"]
+    ratings["religious"] = ratings["Religiosa?"] == "religiosa"
+    ratings.drop("Orientamento politico", axis=1, inplace=True)
+    ratings.drop("Religiosa?", axis=1, inplace=True)
+
+    ratings.to_csv("data/migration/provider_pol_rel_ratings.csv")
+
+
+if __name__ == "__main__":
+    main()

sociofillmore/migration/preprocess.py

@@ -0,0 +1,57 @@
+import pandas as pd
+import numpy as np
+
+import random
+
+random.seed(1996)
+
+
+CORPUS_ANNOTATED = "data/migration/corpus_with_frames_and_orientation.csv"
+CORPUS_ALL = "data/migration/corpus_all.csv"
+
+RATIO_DEV = 0.05
+RATIO_TEST = 0.25
+
+
+def preprocess_annotated():
+    print("Loading corpus...")
+    df = pd.read_csv(CORPUS_ANNOTATED, encoding="latin1")
+    print(f"\tfound {len(df)} annotated headlines")
+
+    train_idx = []
+    dev_idx = []
+    test_idx = []
+
+    print("Making random train/dev/test split...")
+    for i in range(len(df)):
+        rnd = random.random()
+        if rnd < RATIO_DEV:
+            dev_idx.append(i)
+        elif rnd < (RATIO_DEV + RATIO_TEST):
+            test_idx.append(i)
+        else:
+            train_idx.append(i)
+
+    print(f"\tassigned {len(train_idx)} samples to train")
+    print(f"\tassigned {len(dev_idx)} samples to dev")
+    print(f"\tassigned {len(test_idx)} samples to test")
+
+    df_train = df.iloc[train_idx]
+    df_dev = df.iloc[dev_idx]
+    df_test = df.iloc[test_idx]
+
+    df_train.to_csv("output/migration/preprocess/annotations_train.csv")
+    df_dev.to_csv("output/migration/preprocess/annotations_dev.csv")
+    df_test.to_csv("output/migration/preprocess/annotations_test.csv")
+
+
+def preprocess_all():
+    df = pd.read_csv(CORPUS_ANNOTATED, encoding="latin1")
+    for _, row in df.iterrows():
+        pass
+
+
+
+if __name__ == "__main__":
+    # preprocess_annotated()
+    preprocess_all()

sociofillmore/migration/split_data.py

@@ -0,0 +1,85 @@
+import random
+import datetime
+
+import pandas as pd
+
+random.seed(1996)
+
+
+DEV_RATIO = 0.10
+
+
+def choose_best_casing(orig, predicted):
+    num_upper_tokens = len([c == c.upper() for c in orig.upper()])
+    if num_upper_tokens > 0.5 * len(orig):
+        return predicted
+    return predicted
+
+
+def split_data():
+    events_main = []
+    texts_main = []
+    events_dev = []
+    texts_dev = []
+    
+    with open("data/migration/corpus_titoli_all_raw.truecase_bilstm.txt", encoding="utf-8") as f:
+        titles_tc = [line.strip() for line in f]
+
+    df_all = pd.read_csv("data/migration/corpus_all.csv", encoding="latin-1")
+    for idx, (_, row) in enumerate(df_all.iterrows()):
+        
+        if idx % 1000 == 0:
+            print("Processing line:", idx)
+
+        year = int(row["Anno"])
+
+        event_data = {
+            "event:id": idx,
+            "event:year": year,
+    
+        }
+        text_data = {
+            "event_id": idx,
+            "text_id": idx,
+            "pubyear": year,
+            "language": "Italian",
+            "provider": row["Testata"].lstrip("*T_"),
+            "title": choose_best_casing(row["Titolo"], titles_tc[idx]), 
+            "title_truecased": titles_tc[idx],
+            "title_orig": row["Titolo"]
+        }
+
+        if random.random() < DEV_RATIO:
+            events_dev.append(event_data)
+            texts_dev.append(text_data)
+
+            with open(f"output/migration/split_data/split_dev10_sep_txt_files/{idx}.best.txt", "w", encoding="utf-8") as f_out:
+                f_out.write(text_data["title"])
+
+            with open(f"output/migration/split_data/split_dev10_sep_txt_files/{idx}.orig.txt", "w", encoding="utf-8") as f_out:
+                f_out.write(text_data["title_orig"])
+
+            with open(f"output/migration/split_data/split_dev10_sep_txt_files/{idx}.truecase.txt", "w", encoding="utf-8") as f_out:
+                f_out.write(text_data["title_truecased"])
+
+        else:
+            events_main.append(event_data)
+            texts_main.append(text_data)
+
+            with open(f"output/migration/split_data/split_main_sep_txt_files/{idx}.best.txt", "w", encoding="utf-8") as f_out:
+                f_out.write(text_data["title"])
+
+            with open(f"output/migration/split_data/split_main_sep_txt_files/{idx}.orig.txt", "w", encoding="utf-8") as f_out:
+                f_out.write(text_data["title_orig"])
+
+            with open(f"output/migration/split_data/split_main_sep_txt_files/{idx}.truecase.txt", "w", encoding="utf-8") as f_out:
+                f_out.write(text_data["title_truecased"])
+
+    pd.DataFrame(events_main).to_csv("output/migration/split_data/split_main.events.csv")
+    pd.DataFrame(texts_main).to_csv("output/migration/split_data/split_main.texts.meta.csv")
+    pd.DataFrame(events_dev).to_csv("output/migration/split_data/split_dev10.events.csv")
+    pd.DataFrame(texts_dev).to_csv("output/migration/split_data/split_dev10.texts.meta.csv")
+
+
+if __name__ == "__main__":
+    split_data()

sociofillmore/migration/split_lome_predictions.py

@@ -0,0 +1,54 @@
+import os
+import json
+
+import pandas as pd
+
+
+def main(input_json, input_txt, output_dir):
+    meta_df = pd.read_csv("output/migration/split_data/split_dev10.texts.meta.csv")
+    text_ids = meta_df["text_id"].to_list()
+    with open(input_json, encoding="utf-8") as f:
+        json_predictions = json.load(f)
+
+    with open(input_txt, encoding="utf-8") as f:
+        txt_predictions = f.read().split("\n\n")
+
+    for t_id, json_p, txt_p in zip(text_ids, json_predictions, txt_predictions):
+
+        if int(t_id) % 100 == 0:
+            print(t_id)
+
+        prediction_dir = f"{output_dir}/{t_id}"
+        if not os.path.isdir(prediction_dir):
+            os.makedirs(prediction_dir)
+        prediction_file_json = f"{prediction_dir}/lome_{t_id}.comm.json"
+        prediction_file_txt = f"{prediction_dir}/lome_{t_id}.comm.txt"
+
+        with open(prediction_file_json, "w", encoding="utf-8") as f_out:
+            json.dump([json_p], f_out)
+
+        with open(prediction_file_txt, "w", encoding="utf-8") as f_out:
+            f_out.write(txt_p + "\n\n")
+
+
+if __name__ == "__main__":
+    # main(
+    #     input_json="output/migration/lome/lome_0shot/lome_lome_0shot_migration_all_tc.comm.json",
+    #     input_txt="output/migration/lome/lome_0shot/lome_lome_0shot_migration_all_tc.comm.txt",
+    #     output_dir="output/migration/lome/multilabel/lome_0shot/pavia"
+    # )
+    # main(
+    #     input_json="output/migration/lome/lome_0shot/lome_lome_0shot_migration_all_best-truecase.comm.json",
+    #     input_txt="output/migration/lome/lome_0shot/lome_lome_0shot_migration_all_best-truecase.comm.txt",
+    #     output_dir="output/migration/lome/multilabel/lome_0shot/pavia"
+    # )
+    # main(
+    #     input_json="output/migration/lome/lome_zs-tgt_ev-frm/data-in.concat.combined_zs_ev.tc_bilstm.json",
+    #     input_txt="output/migration/lome/lome_zs-tgt_ev-frm/data-in.concat.combined_zs_ev.tc_bilstm.txt",
+    #     output_dir="output/migration/lome/multilabel/lome_zs-tgt_ev_frm/pavia"
+    # )
+    main(
+        input_json="/home/gossminn/WorkSyncs/Code/fn-for-social-frames/output/migration/lome/lome_migration_concat.comm.json", 
+        input_txt="/home/gossminn/WorkSyncs/Code/fn-for-social-frames/output/migration/lome/lome_migration_concat.comm.txt",
+        output_dir="output/migration/lome/multilabel/lome_0shot/pavia"
+    )