🌑 Promethean Dawn 26B-A4B

This is a merge of pre-trained language models created using mergekit-exp.

This model was merged using the moe_della merge method, and combines every ReadyArt 26B model.

architecture: Gemma4ForConditionalGeneration
base_model: B:\26B\google--gemma-4-26B-A4B-it
models:
  - model: B:\26B\ReadyArt--Dark-Scarlett-v1.0-26B-A4B
    parameters:
      weight:
        - filter: "lm_head"
          value: 1.0
        - filter: "attn"
          value: 1.0
        - filter: "embed_tokens"
          value: 1.0
        - value: 1.0
      density: 0.9
      epsilon: 0.09
  - model: B:\26B\ReadyArt--For-Her-Darkside-26B-A4B-v1.4
    parameters:
      weight:
        - filter: "lm_head"
          value: 1.0
        - filter: "attn"
          value: 1.0
        - filter: "embed_tokens"
          value: 1.0
        - value: 1.0
      density: 0.9
      epsilon: 0.09
  - model: B:\26B\ReadyArt--Heimdallr-26B-A4B-v0.35-Q8_0
    parameters:
      weight:
        - filter: "lm_head"
          value: 1.0
        - filter: "attn"
          value: 1.0
        - filter: "embed_tokens"
          value: 1.0
        - value: 1.0
      density: 0.9
      epsilon: 0.09
  - model: B:\26B\ReadyArt--Melody1437-26B-A4B-v2.0
    parameters:
      weight:
        - filter: "lm_head"
          value: 1.0
        - filter: "attn"
          value: 1.0
        - filter: "embed_tokens"
          value: 1.0
        - value: 1.0
      density: 0.9
      epsilon: 0.09
  - model: B:\26B\ReadyArt--Omega-Evolution-26B-A4B-v3.0-Q8_0
    parameters:
      weight:
        - filter: "lm_head"
          value: 1.0
        - filter: "attn"
          value: 1.0
        - filter: "embed_tokens"
          value: 1.0
        - value: 1.0
      density: 0.9
      epsilon: 0.09
  - model: B:\26B\ReadyArt--Serenity-26B-A4B
    parameters:
      weight:
        - filter: "lm_head"
          value: 1.0
        - filter: "attn"
          value: 1.0
        - filter: "embed_tokens"
          value: 1.0
        - value: 1.0
      density: 0.9
      epsilon: 0.09
merge_method: moe_della
parameters:  
  lambda: 1.0
  # normalize: true # deprecated
  normalize_weights: true
  normalize_router: true
  int8_mask: false
  rescale: true
  router_strategy: della # average # random_init
  blend_experts: true
dtype: float32
out_dtype: bfloat16
tokenizer:
  source: union
chat_template: auto
name: 🌑 Promethean Dawn 26B-A4B