ugc rel tag and filter during centrality calculation

StractOrg · Nov 20, 2024 · 0681588 · 0681588
1 parent d2fa5f4
commit 0681588
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 0 deletions.
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
@@ -41,6 +41,7 @@ pub static SKIPPED_REL: std::sync::LazyLock<RelFlags> = std::sync::LazyLock::new
         | RelFlags::SEARCH
         | RelFlags::LINK_TAG
         | RelFlags::SCRIPT_TAG
+        | RelFlags::UGC
 });
 
 #[derive(Debug, Clone, bincode::Decode, bincode::Encode)]

diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rs
@@ -45,6 +45,7 @@ pub static SKIPPED_REL: std::sync::LazyLock<RelFlags> = std::sync::LazyLock::new
         | RelFlags::LINK_TAG
         | RelFlags::SCRIPT_TAG
         | RelFlags::SAME_ICANN_DOMAIN
+        | RelFlags::UGC
 });
 
 type Counter = BTreeMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>>;

diff --git a/crates/core/src/webpage/html/links.rs b/crates/core/src/webpage/html/links.rs
@@ -82,6 +82,7 @@ impl RelFlags {
                     "tag" => res |= RelFlags::TAG,
                     "terms-of-service" => res |= RelFlags::TERMS_OF_SERVICE,
                     "sponsored" => res |= RelFlags::SPONSORED,
+                    "ugc" => res |= RelFlags::UGC,
                     _ => {}
                 }
             }
@@ -135,6 +136,7 @@ bitflags! {
         const SCRIPT_TAG = 1 << 19;
         const META_TAG = 1 << 20;
         const SAME_ICANN_DOMAIN = 1 << 21;
+        const UGC = 1 << 22;
     }
 }