Add README
Browse files
README.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
- Base backbone: `revliter/internvideo_next_large_p14_res224_f16`
|
| 4 |
- Clip length: `16` frames
|
| 5 |
- Frame size: `224x224`
|
| 6 |
-
- Head hidden dims: `512`
|
| 7 |
- Repo: `happy8825/internvideo_tuned`
|
| 8 |
|
| 9 |
## Quick start (single video)
|
|
@@ -15,6 +15,60 @@ python inference_example.py --repo_id happy8825/internvideo_tuned --video /path/
|
|
| 15 |
|
| 16 |
The script downloads this repo, loads the InternVideo backbone + tuned head, and prints `normal` or `abnormal`.
|
| 17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
## Files
|
| 19 |
- `best_head.pt`: classifier head weights
|
| 20 |
- `train_config.json`: training config (contains base model, clip_len, frame_size, hidden dims, etc.)
|
|
|
|
| 3 |
- Base backbone: `revliter/internvideo_next_large_p14_res224_f16`
|
| 4 |
- Clip length: `16` frames
|
| 5 |
- Frame size: `224x224`
|
| 6 |
+
- Head hidden dims: `[512]`
|
| 7 |
- Repo: `happy8825/internvideo_tuned`
|
| 8 |
|
| 9 |
## Quick start (single video)
|
|
|
|
| 15 |
|
| 16 |
The script downloads this repo, loads the InternVideo backbone + tuned head, and prints `normal` or `abnormal`.
|
| 17 |
|
| 18 |
+
## Minimal Python snippet
|
| 19 |
+
|
| 20 |
+
```python
|
| 21 |
+
import json, os, numpy as np, torch
|
| 22 |
+
from huggingface_hub import snapshot_download
|
| 23 |
+
from transformers import VideoMAEImageProcessor, AutoModel
|
| 24 |
+
from decord import VideoReader
|
| 25 |
+
|
| 26 |
+
ID2LABEL = {0: "normal", 1: "abnormal"}
|
| 27 |
+
|
| 28 |
+
class ClassificationHead(torch.nn.Module):
|
| 29 |
+
def __init__(self, in_dim, hidden_dims, num_labels=2, dropout=0.1):
|
| 30 |
+
super().__init__()
|
| 31 |
+
dims = [in_dim] + list(hidden_dims)
|
| 32 |
+
layers = []
|
| 33 |
+
for i in range(len(dims) - 1):
|
| 34 |
+
layers += [torch.nn.Linear(dims[i], dims[i+1]), torch.nn.GELU(), torch.nn.Dropout(dropout)]
|
| 35 |
+
layers.append(torch.nn.Linear(dims[-1], num_labels))
|
| 36 |
+
self.net = torch.nn.Sequential(*layers)
|
| 37 |
+
def forward(self, x): return self.net(x)
|
| 38 |
+
|
| 39 |
+
def pool_tokens(feats, expected=None):
|
| 40 |
+
if feats.dim() != 3: return feats
|
| 41 |
+
_, d1, d2 = feats.shape
|
| 42 |
+
if expected:
|
| 43 |
+
if d1 == expected: return feats.mean(dim=2)
|
| 44 |
+
if d2 == expected: return feats.mean(dim=1)
|
| 45 |
+
return feats.mean(dim=2 if d1 <= d2 else 1)
|
| 46 |
+
|
| 47 |
+
repo = "happy8825/internvideo_tuned"
|
| 48 |
+
local = snapshot_download(repo)
|
| 49 |
+
cfg = json.load(open(os.path.join(local, "train_config.json")))
|
| 50 |
+
base = cfg.get("base_model", "revliter/internvideo_next_large_p14_res224_f16")
|
| 51 |
+
clip_len = int(cfg.get("clip_len", 16))
|
| 52 |
+
hidden = cfg.get("hidden", [512])
|
| 53 |
+
feat_dim = cfg.get("feature_dim") or cfg.get("hidden_size")
|
| 54 |
+
|
| 55 |
+
processor = VideoMAEImageProcessor.from_pretrained(base)
|
| 56 |
+
backbone = AutoModel.from_pretrained(base, trust_remote_code=True).eval().to("cuda")
|
| 57 |
+
head = ClassificationHead(in_dim=feat_dim or backbone.config.hidden_size, hidden_dims=hidden)
|
| 58 |
+
state = torch.load(os.path.join(local, "best_head.pt"), map_location="cpu")
|
| 59 |
+
head.load_state_dict(state["head"]); head.eval().to("cuda")
|
| 60 |
+
|
| 61 |
+
vr = VideoReader("/path/to/video.mp4")
|
| 62 |
+
idxs = np.linspace(0, len(vr)-1, num=clip_len, dtype=int)
|
| 63 |
+
frames = [vr[i].asnumpy() for i in idxs]
|
| 64 |
+
px = processor(frames, return_tensors="pt")["pixel_values"].permute(0,2,1,3,4).to("cuda")
|
| 65 |
+
with torch.no_grad(), torch.amp.autocast("cuda", dtype=torch.bfloat16):
|
| 66 |
+
feats = backbone.extract_features(pixel_values=px)
|
| 67 |
+
pooled = pool_tokens(feats, expected=feat_dim)
|
| 68 |
+
pred = int(head(pooled.float()).argmax(dim=-1).item())
|
| 69 |
+
print(ID2LABEL.get(pred, pred))
|
| 70 |
+
```
|
| 71 |
+
|
| 72 |
## Files
|
| 73 |
- `best_head.pt`: classifier head weights
|
| 74 |
- `train_config.json`: training config (contains base model, clip_len, frame_size, hidden dims, etc.)
|