Commit | Line | Data |
---|---|---|
da7c5cc6 | 1 | /* |
5b519e94 | 2 | * All rights reserved. |
da7c5cc6 | 3 | * |
dbf0c423 | 4 | * %sccs.include.redist.c% |
5b519e94 | 5 | * |
9db58063 | 6 | * @(#)uipc_mbuf.c 7.18 (Berkeley) %G% |
da7c5cc6 | 7 | */ |
961945a8 | 8 | |
94368568 | 9 | #include "param.h" |
94368568 JB |
10 | #include "user.h" |
11 | #include "proc.h" | |
12 | #include "cmap.h" | |
cb80fe27 | 13 | #include "malloc.h" |
94368568 | 14 | #include "map.h" |
cb80fe27 | 15 | #define MBTYPES |
94368568 | 16 | #include "mbuf.h" |
94368568 | 17 | #include "kernel.h" |
6d0e638a MK |
18 | #include "syslog.h" |
19 | #include "domain.h" | |
20 | #include "protosw.h" | |
9db58063 KM |
21 | #include "../vm/vm_param.h" |
22 | #include "../vm/vm_map.h" | |
23 | ||
24 | extern vm_map_t mb_map; | |
25 | struct mbuf *mbutl; | |
26 | char *mclrefcnt; | |
d80cae33 | 27 | |
7c733634 BJ |
28 | mbinit() |
29 | { | |
9c59be2b | 30 | int s; |
7c733634 | 31 | |
9db58063 | 32 | #if CLBYTES < 4096 |
d7b48ae0 | 33 | #define NCL_INIT (4096/CLBYTES) |
a3010bc1 MK |
34 | #else |
35 | #define NCL_INIT 1 | |
36 | #endif | |
9c59be2b | 37 | s = splimp(); |
cb80fe27 | 38 | if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0) |
7c733634 | 39 | goto bad; |
9c59be2b | 40 | splx(s); |
7c733634 BJ |
41 | return; |
42 | bad: | |
43 | panic("mbinit"); | |
44 | } | |
45 | ||
9c59be2b | 46 | /* |
cb80fe27 MK |
47 | * Allocate some number of mbuf clusters |
48 | * and place on cluster free list. | |
9c59be2b MK |
49 | * Must be called at splimp. |
50 | */ | |
6d0e638a | 51 | /* ARGSUSED */ |
cb80fe27 | 52 | m_clalloc(ncl, canwait) |
7c733634 | 53 | register int ncl; |
7c733634 BJ |
54 | { |
55 | int npg, mbx; | |
cb80fe27 | 56 | register caddr_t p; |
7c733634 | 57 | register int i; |
6d0e638a | 58 | static int logged; |
7c733634 | 59 | |
7c733634 | 60 | npg = ncl * CLSIZE; |
9db58063 KM |
61 | p = (caddr_t)kmem_malloc(mb_map, ctob(npg), canwait); |
62 | if (p == NULL) { | |
6d0e638a MK |
63 | if (logged == 0) { |
64 | logged++; | |
9db58063 | 65 | log(LOG_ERR, "mb_map full\n"); |
6d0e638a | 66 | } |
7c733634 | 67 | return (0); |
9c59be2b | 68 | } |
cb80fe27 MK |
69 | ncl = ncl * CLBYTES / MCLBYTES; |
70 | for (i = 0; i < ncl; i++) { | |
71 | ((union mcluster *)p)->mcl_next = mclfree; | |
72 | mclfree = (union mcluster *)p; | |
73 | p += MCLBYTES; | |
74 | mbstat.m_clfree++; | |
7c733634 | 75 | } |
cb80fe27 MK |
76 | mbstat.m_clusters += ncl; |
77 | return (1); | |
7c733634 BJ |
78 | } |
79 | ||
9c59be2b | 80 | /* |
cb80fe27 MK |
81 | * When MGET failes, ask protocols to free space when short of memory, |
82 | * then re-attempt to allocate an mbuf. | |
83 | */ | |
84 | struct mbuf * | |
85 | m_retry(i, t) | |
86 | int i, t; | |
87 | { | |
88 | register struct mbuf *m; | |
89 | ||
90 | m_reclaim(); | |
91 | #define m_retry(i, t) (struct mbuf *)0 | |
92 | MGET(m, i, t); | |
93 | #undef m_retry | |
94 | return (m); | |
95 | } | |
96 | ||
97 | /* | |
98 | * As above; retry an MGETHDR. | |
9c59be2b | 99 | */ |
cb80fe27 MK |
100 | struct mbuf * |
101 | m_retryhdr(i, t) | |
102 | int i, t; | |
103 | { | |
104 | register struct mbuf *m; | |
105 | ||
106 | m_reclaim(); | |
107 | #define m_retryhdr(i, t) (struct mbuf *)0 | |
108 | MGETHDR(m, i, t); | |
109 | #undef m_retryhdr | |
110 | return (m); | |
111 | } | |
112 | ||
113 | m_reclaim() | |
7c733634 | 114 | { |
6d0e638a MK |
115 | register struct domain *dp; |
116 | register struct protosw *pr; | |
cb80fe27 | 117 | int s = splimp(); |
7c733634 | 118 | |
cb80fe27 MK |
119 | for (dp = domains; dp; dp = dp->dom_next) |
120 | for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) | |
121 | if (pr->pr_drain) | |
122 | (*pr->pr_drain)(); | |
123 | splx(s); | |
124 | mbstat.m_drain++; | |
125 | } | |
7c733634 | 126 | |
7c733634 BJ |
127 | /* |
128 | * Space allocation routines. | |
129 | * These are also available as macros | |
130 | * for critical paths. | |
131 | */ | |
d80cae33 | 132 | struct mbuf * |
cce93e4b SL |
133 | m_get(canwait, type) |
134 | int canwait, type; | |
d80cae33 BJ |
135 | { |
136 | register struct mbuf *m; | |
137 | ||
cce93e4b | 138 | MGET(m, canwait, type); |
d80cae33 BJ |
139 | return (m); |
140 | } | |
141 | ||
cb80fe27 MK |
142 | struct mbuf * |
143 | m_gethdr(canwait, type) | |
144 | int canwait, type; | |
145 | { | |
146 | register struct mbuf *m; | |
147 | ||
148 | MGETHDR(m, canwait, type); | |
149 | return (m); | |
150 | } | |
151 | ||
cc15ab5d | 152 | struct mbuf * |
cce93e4b SL |
153 | m_getclr(canwait, type) |
154 | int canwait, type; | |
cc15ab5d BJ |
155 | { |
156 | register struct mbuf *m; | |
157 | ||
9c59be2b | 158 | MGET(m, canwait, type); |
cc15ab5d BJ |
159 | if (m == 0) |
160 | return (0); | |
cc15ab5d BJ |
161 | bzero(mtod(m, caddr_t), MLEN); |
162 | return (m); | |
163 | } | |
164 | ||
d80cae33 BJ |
165 | struct mbuf * |
166 | m_free(m) | |
167 | struct mbuf *m; | |
168 | { | |
169 | register struct mbuf *n; | |
170 | ||
d80cae33 BJ |
171 | MFREE(m, n); |
172 | return (n); | |
173 | } | |
174 | ||
8f3e7457 | 175 | m_freem(m) |
d80cae33 BJ |
176 | register struct mbuf *m; |
177 | { | |
178 | register struct mbuf *n; | |
d80cae33 | 179 | |
d80cae33 | 180 | if (m == NULL) |
ae921915 | 181 | return; |
d80cae33 | 182 | do { |
8f3e7457 | 183 | MFREE(m, n); |
d80cae33 | 184 | } while (m = n); |
2b4b57cd BJ |
185 | } |
186 | ||
7c733634 BJ |
187 | /* |
188 | * Mbuffer utility routines. | |
189 | */ | |
9c59be2b | 190 | |
39efebb8 | 191 | /* |
cb80fe27 MK |
192 | * Lesser-used path for M_PREPEND: |
193 | * allocate new mbuf to prepend to chain, | |
194 | * copy junk along. | |
195 | */ | |
196 | struct mbuf * | |
197 | m_prepend(m, len, how) | |
198 | register struct mbuf *m; | |
199 | int len, how; | |
200 | { | |
201 | struct mbuf *mn; | |
202 | ||
203 | MGET(mn, how, m->m_type); | |
204 | if (mn == (struct mbuf *)NULL) { | |
205 | m_freem(m); | |
206 | return ((struct mbuf *)NULL); | |
207 | } | |
208 | if (m->m_flags & M_PKTHDR) { | |
209 | M_COPY_PKTHDR(mn, m); | |
210 | m->m_flags &= ~M_PKTHDR; | |
211 | } | |
212 | mn->m_next = m; | |
213 | m = mn; | |
214 | if (len < MHLEN) | |
215 | MH_ALIGN(m, len); | |
216 | m->m_len = len; | |
217 | return (m); | |
218 | } | |
219 | ||
cb80fe27 MK |
220 | /* |
221 | * Make a copy of an mbuf chain starting "off0" bytes from the beginning, | |
9c59be2b | 222 | * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. |
cb80fe27 | 223 | * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. |
9c59be2b | 224 | */ |
33b10a63 KM |
225 | int MCFail; |
226 | ||
2b4b57cd | 227 | struct mbuf * |
cb80fe27 | 228 | m_copym(m, off0, len, wait) |
2b4b57cd | 229 | register struct mbuf *m; |
cb80fe27 | 230 | int off0, wait; |
2b4b57cd BJ |
231 | register int len; |
232 | { | |
233 | register struct mbuf *n, **np; | |
cb80fe27 MK |
234 | register int off = off0; |
235 | struct mbuf *top; | |
236 | int copyhdr = 0; | |
2b4b57cd | 237 | |
2b4b57cd | 238 | if (off < 0 || len < 0) |
cb80fe27 MK |
239 | panic("m_copym"); |
240 | if (off == 0 && m->m_flags & M_PKTHDR) | |
241 | copyhdr = 1; | |
2b4b57cd BJ |
242 | while (off > 0) { |
243 | if (m == 0) | |
cb80fe27 | 244 | panic("m_copym"); |
2b4b57cd BJ |
245 | if (off < m->m_len) |
246 | break; | |
247 | off -= m->m_len; | |
248 | m = m->m_next; | |
249 | } | |
250 | np = ⊤ | |
251 | top = 0; | |
252 | while (len > 0) { | |
842ff042 BJ |
253 | if (m == 0) { |
254 | if (len != M_COPYALL) | |
cb80fe27 | 255 | panic("m_copym"); |
842ff042 BJ |
256 | break; |
257 | } | |
cb80fe27 | 258 | MGET(n, wait, m->m_type); |
2b4b57cd BJ |
259 | *np = n; |
260 | if (n == 0) | |
261 | goto nospace; | |
cb80fe27 MK |
262 | if (copyhdr) { |
263 | M_COPY_PKTHDR(n, m); | |
264 | if (len == M_COPYALL) | |
265 | n->m_pkthdr.len -= off0; | |
266 | else | |
267 | n->m_pkthdr.len = len; | |
268 | copyhdr = 0; | |
269 | } | |
2b4b57cd | 270 | n->m_len = MIN(len, m->m_len - off); |
cb80fe27 MK |
271 | if (m->m_flags & M_EXT) { |
272 | n->m_data = m->m_data + off; | |
273 | mclrefcnt[mtocl(m->m_ext.ext_buf)]++; | |
274 | n->m_ext = m->m_ext; | |
275 | n->m_flags |= M_EXT; | |
970108c7 | 276 | } else |
2b4b57cd BJ |
277 | bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), |
278 | (unsigned)n->m_len); | |
842ff042 BJ |
279 | if (len != M_COPYALL) |
280 | len -= n->m_len; | |
2b4b57cd BJ |
281 | off = 0; |
282 | m = m->m_next; | |
283 | np = &n->m_next; | |
284 | } | |
33b10a63 KM |
285 | if (top == 0) |
286 | MCFail++; | |
2b4b57cd BJ |
287 | return (top); |
288 | nospace: | |
2b4b57cd | 289 | m_freem(top); |
33b10a63 | 290 | MCFail++; |
2b4b57cd | 291 | return (0); |
d80cae33 BJ |
292 | } |
293 | ||
39efebb8 MK |
294 | /* |
295 | * Copy data from an mbuf chain starting "off" bytes from the beginning, | |
296 | * continuing for "len" bytes, into the indicated buffer. | |
297 | */ | |
39efebb8 MK |
298 | m_copydata(m, off, len, cp) |
299 | register struct mbuf *m; | |
a2ec3cef | 300 | register int off; |
39efebb8 | 301 | register int len; |
a2ec3cef | 302 | caddr_t cp; |
39efebb8 MK |
303 | { |
304 | register unsigned count; | |
305 | ||
306 | if (off < 0 || len < 0) | |
307 | panic("m_copydata"); | |
308 | while (off > 0) { | |
309 | if (m == 0) | |
310 | panic("m_copydata"); | |
311 | if (off < m->m_len) | |
312 | break; | |
313 | off -= m->m_len; | |
314 | m = m->m_next; | |
315 | } | |
316 | while (len > 0) { | |
317 | if (m == 0) | |
318 | panic("m_copydata"); | |
a2ec3cef | 319 | count = MIN(m->m_len - off, len); |
39efebb8 MK |
320 | bcopy(mtod(m, caddr_t) + off, cp, count); |
321 | len -= count; | |
a2ec3cef | 322 | cp += count; |
39efebb8 MK |
323 | off = 0; |
324 | m = m->m_next; | |
325 | } | |
326 | } | |
327 | ||
cb80fe27 MK |
328 | /* |
329 | * Concatenate mbuf chain n to m. | |
330 | * Both chains must be of the same type (e.g. MT_DATA). | |
331 | * Any m_pkthdr is not updated. | |
332 | */ | |
8f3e7457 BJ |
333 | m_cat(m, n) |
334 | register struct mbuf *m, *n; | |
335 | { | |
8f3e7457 BJ |
336 | while (m->m_next) |
337 | m = m->m_next; | |
e495e1cc | 338 | while (n) { |
cb80fe27 MK |
339 | if (m->m_flags & M_EXT || |
340 | m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { | |
e495e1cc | 341 | /* just join the two chains */ |
8f3e7457 | 342 | m->m_next = n; |
e495e1cc | 343 | return; |
8f3e7457 | 344 | } |
e495e1cc BJ |
345 | /* splat the data from one into the other */ |
346 | bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, | |
347 | (u_int)n->m_len); | |
348 | m->m_len += n->m_len; | |
349 | n = m_free(n); | |
350 | } | |
8f3e7457 BJ |
351 | } |
352 | ||
cb80fe27 | 353 | m_adj(mp, req_len) |
d80cae33 | 354 | struct mbuf *mp; |
d80cae33 | 355 | { |
cb80fe27 | 356 | register int len = req_len; |
fcd25562 MK |
357 | register struct mbuf *m; |
358 | register count; | |
d80cae33 | 359 | |
d80cae33 BJ |
360 | if ((m = mp) == NULL) |
361 | return; | |
6d19f7ef | 362 | if (len >= 0) { |
cb80fe27 MK |
363 | /* |
364 | * Trim from head. | |
365 | */ | |
d80cae33 | 366 | while (m != NULL && len > 0) { |
6d19f7ef | 367 | if (m->m_len <= len) { |
d80cae33 BJ |
368 | len -= m->m_len; |
369 | m->m_len = 0; | |
370 | m = m->m_next; | |
6d19f7ef | 371 | } else { |
d80cae33 | 372 | m->m_len -= len; |
cb80fe27 MK |
373 | m->m_data += len; |
374 | len = 0; | |
d80cae33 BJ |
375 | } |
376 | } | |
cb80fe27 MK |
377 | m = mp; |
378 | if (mp->m_flags & M_PKTHDR) | |
379 | m->m_pkthdr.len -= (req_len - len); | |
6d19f7ef | 380 | } else { |
fcd25562 MK |
381 | /* |
382 | * Trim from tail. Scan the mbuf chain, | |
383 | * calculating its length and finding the last mbuf. | |
384 | * If the adjustment only affects this mbuf, then just | |
385 | * adjust and return. Otherwise, rescan and truncate | |
386 | * after the remaining size. | |
387 | */ | |
d80cae33 | 388 | len = -len; |
fcd25562 MK |
389 | count = 0; |
390 | for (;;) { | |
391 | count += m->m_len; | |
392 | if (m->m_next == (struct mbuf *)0) | |
393 | break; | |
394 | m = m->m_next; | |
395 | } | |
396 | if (m->m_len >= len) { | |
397 | m->m_len -= len; | |
698e6480 KS |
398 | if ((mp = m)->m_flags & M_PKTHDR) |
399 | m->m_pkthdr.len -= len; | |
fcd25562 MK |
400 | return; |
401 | } | |
402 | count -= len; | |
cb80fe27 MK |
403 | if (count < 0) |
404 | count = 0; | |
fcd25562 MK |
405 | /* |
406 | * Correct length for chain is "count". | |
407 | * Find the mbuf with last data, adjust its length, | |
408 | * and toss data from remaining mbufs on chain. | |
409 | */ | |
cb80fe27 MK |
410 | m = mp; |
411 | if (m->m_flags & M_PKTHDR) | |
412 | m->m_pkthdr.len = count; | |
413 | for (; m; m = m->m_next) { | |
fcd25562 MK |
414 | if (m->m_len >= count) { |
415 | m->m_len = count; | |
d80cae33 BJ |
416 | break; |
417 | } | |
fcd25562 | 418 | count -= m->m_len; |
d80cae33 | 419 | } |
fcd25562 MK |
420 | while (m = m->m_next) |
421 | m->m_len = 0; | |
d80cae33 BJ |
422 | } |
423 | } | |
7c733634 | 424 | |
864d7180 MK |
425 | /* |
426 | * Rearange an mbuf chain so that len bytes are contiguous | |
427 | * and in the data area of an mbuf (so that mtod and dtom | |
fcd25562 MK |
428 | * will work for a structure of size len). Returns the resulting |
429 | * mbuf chain on success, frees it and returns null on failure. | |
cb80fe27 | 430 | * If there is room, it will add up to max_protohdr-len extra bytes to the |
fcd25562 | 431 | * contiguous region in an attempt to avoid being called next time. |
864d7180 | 432 | */ |
33b10a63 KM |
433 | int MPFail; |
434 | ||
10103653 | 435 | struct mbuf * |
fcd25562 MK |
436 | m_pullup(n, len) |
437 | register struct mbuf *n; | |
7c733634 BJ |
438 | int len; |
439 | { | |
fcd25562 MK |
440 | register struct mbuf *m; |
441 | register int count; | |
442 | int space; | |
7c733634 | 443 | |
cb80fe27 MK |
444 | /* |
445 | * If first mbuf has no cluster, and has room for len bytes | |
446 | * without shifting current data, pullup into it, | |
447 | * otherwise allocate a new mbuf to prepend to the chain. | |
448 | */ | |
449 | if ((n->m_flags & M_EXT) == 0 && | |
450 | n->m_data + len < &n->m_dat[MLEN] && n->m_next) { | |
451 | if (n->m_len >= len) | |
452 | return (n); | |
fcd25562 MK |
453 | m = n; |
454 | n = n->m_next; | |
455 | len -= m->m_len; | |
456 | } else { | |
cb80fe27 | 457 | if (len > MHLEN) |
fcd25562 MK |
458 | goto bad; |
459 | MGET(m, M_DONTWAIT, n->m_type); | |
460 | if (m == 0) | |
461 | goto bad; | |
462 | m->m_len = 0; | |
69e969b0 | 463 | if (n->m_flags & M_PKTHDR) { |
cb80fe27 | 464 | M_COPY_PKTHDR(m, n); |
69e969b0 MK |
465 | n->m_flags &= ~M_PKTHDR; |
466 | } | |
fcd25562 | 467 | } |
cb80fe27 | 468 | space = &m->m_dat[MLEN] - (m->m_data + m->m_len); |
10103653 | 469 | do { |
cb80fe27 MK |
470 | count = min(min(max(len, max_protohdr), space), n->m_len); |
471 | bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, | |
668cc26d SL |
472 | (unsigned)count); |
473 | len -= count; | |
474 | m->m_len += count; | |
668cc26d | 475 | n->m_len -= count; |
cb80fe27 | 476 | space -= count; |
10103653 | 477 | if (n->m_len) |
cb80fe27 | 478 | n->m_data += count; |
864d7180 MK |
479 | else |
480 | n = m_free(n); | |
fcd25562 MK |
481 | } while (len > 0 && n); |
482 | if (len > 0) { | |
10103653 BJ |
483 | (void) m_free(m); |
484 | goto bad; | |
485 | } | |
486 | m->m_next = n; | |
487 | return (m); | |
488 | bad: | |
82bff71e | 489 | m_freem(n); |
33b10a63 | 490 | MPFail++; |
7c733634 BJ |
491 | return (0); |
492 | } |