diff options
Diffstat (limited to 'arch/arc/mm/cache.c')
-rw-r--r-- | arch/arc/mm/cache.c | 111 |
1 files changed, 98 insertions, 13 deletions
diff --git a/arch/arc/mm/cache.c b/arch/arc/mm/cache.c index 928562967f3c..a867575a758b 100644 --- a/arch/arc/mm/cache.c +++ b/arch/arc/mm/cache.c | |||
@@ -21,6 +21,10 @@ | |||
21 | #include <asm/cachectl.h> | 21 | #include <asm/cachectl.h> |
22 | #include <asm/setup.h> | 22 | #include <asm/setup.h> |
23 | 23 | ||
24 | #ifdef CONFIG_ISA_ARCV2 | ||
25 | #define USE_RGN_FLSH 1 | ||
26 | #endif | ||
27 | |||
24 | static int l2_line_sz; | 28 | static int l2_line_sz; |
25 | static int ioc_exists; | 29 | static int ioc_exists; |
26 | int slc_enable = 1, ioc_enable = 1; | 30 | int slc_enable = 1, ioc_enable = 1; |
@@ -28,7 +32,7 @@ unsigned long perip_base = ARC_UNCACHED_ADDR_SPACE; /* legacy value for boot */ | |||
28 | unsigned long perip_end = 0xFFFFFFFF; /* legacy value */ | 32 | unsigned long perip_end = 0xFFFFFFFF; /* legacy value */ |
29 | 33 | ||
30 | void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr, | 34 | void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr, |
31 | unsigned long sz, const int cacheop); | 35 | unsigned long sz, const int op, const int full_page); |
32 | 36 | ||
33 | void (*__dma_cache_wback_inv)(phys_addr_t start, unsigned long sz); | 37 | void (*__dma_cache_wback_inv)(phys_addr_t start, unsigned long sz); |
34 | void (*__dma_cache_inv)(phys_addr_t start, unsigned long sz); | 38 | void (*__dma_cache_inv)(phys_addr_t start, unsigned long sz); |
@@ -233,11 +237,10 @@ slc_chk: | |||
233 | 237 | ||
234 | static inline | 238 | static inline |
235 | void __cache_line_loop_v2(phys_addr_t paddr, unsigned long vaddr, | 239 | void __cache_line_loop_v2(phys_addr_t paddr, unsigned long vaddr, |
236 | unsigned long sz, const int op) | 240 | unsigned long sz, const int op, const int full_page) |
237 | { | 241 | { |
238 | unsigned int aux_cmd; | 242 | unsigned int aux_cmd; |
239 | int num_lines; | 243 | int num_lines; |
240 | const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; | ||
241 | 244 | ||
242 | if (op == OP_INV_IC) { | 245 | if (op == OP_INV_IC) { |
243 | aux_cmd = ARC_REG_IC_IVIL; | 246 | aux_cmd = ARC_REG_IC_IVIL; |
@@ -279,11 +282,10 @@ void __cache_line_loop_v2(phys_addr_t paddr, unsigned long vaddr, | |||
279 | */ | 282 | */ |
280 | static inline | 283 | static inline |
281 | void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr, | 284 | void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr, |
282 | unsigned long sz, const int op) | 285 | unsigned long sz, const int op, const int full_page) |
283 | { | 286 | { |
284 | unsigned int aux_cmd, aux_tag; | 287 | unsigned int aux_cmd, aux_tag; |
285 | int num_lines; | 288 | int num_lines; |
286 | const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; | ||
287 | 289 | ||
288 | if (op == OP_INV_IC) { | 290 | if (op == OP_INV_IC) { |
289 | aux_cmd = ARC_REG_IC_IVIL; | 291 | aux_cmd = ARC_REG_IC_IVIL; |
@@ -334,6 +336,8 @@ void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr, | |||
334 | } | 336 | } |
335 | } | 337 | } |
336 | 338 | ||
339 | #ifndef USE_RGN_FLSH | ||
340 | |||
337 | /* | 341 | /* |
338 | * In HS38x (MMU v4), I-cache is VIPT (can alias), D-cache is PIPT | 342 | * In HS38x (MMU v4), I-cache is VIPT (can alias), D-cache is PIPT |
339 | * Here's how cache ops are implemented | 343 | * Here's how cache ops are implemented |
@@ -349,17 +353,16 @@ void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr, | |||
349 | */ | 353 | */ |
350 | static inline | 354 | static inline |
351 | void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, | 355 | void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, |
352 | unsigned long sz, const int cacheop) | 356 | unsigned long sz, const int op, const int full_page) |
353 | { | 357 | { |
354 | unsigned int aux_cmd; | 358 | unsigned int aux_cmd; |
355 | int num_lines; | 359 | int num_lines; |
356 | const int full_page_op = __builtin_constant_p(sz) && sz == PAGE_SIZE; | ||
357 | 360 | ||
358 | if (cacheop == OP_INV_IC) { | 361 | if (op == OP_INV_IC) { |
359 | aux_cmd = ARC_REG_IC_IVIL; | 362 | aux_cmd = ARC_REG_IC_IVIL; |
360 | } else { | 363 | } else { |
361 | /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ | 364 | /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ |
362 | aux_cmd = cacheop & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; | 365 | aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; |
363 | } | 366 | } |
364 | 367 | ||
365 | /* Ensure we properly floor/ceil the non-line aligned/sized requests | 368 | /* Ensure we properly floor/ceil the non-line aligned/sized requests |
@@ -368,7 +371,7 @@ void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, | |||
368 | * -@paddr will be cache-line aligned already (being page aligned) | 371 | * -@paddr will be cache-line aligned already (being page aligned) |
369 | * -@sz will be integral multiple of line size (being page sized). | 372 | * -@sz will be integral multiple of line size (being page sized). |
370 | */ | 373 | */ |
371 | if (!full_page_op) { | 374 | if (!full_page) { |
372 | sz += paddr & ~CACHE_LINE_MASK; | 375 | sz += paddr & ~CACHE_LINE_MASK; |
373 | paddr &= CACHE_LINE_MASK; | 376 | paddr &= CACHE_LINE_MASK; |
374 | } | 377 | } |
@@ -381,7 +384,7 @@ void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, | |||
381 | * - (and needs to be written before the lower 32 bits) | 384 | * - (and needs to be written before the lower 32 bits) |
382 | */ | 385 | */ |
383 | if (is_pae40_enabled()) { | 386 | if (is_pae40_enabled()) { |
384 | if (cacheop == OP_INV_IC) | 387 | if (op == OP_INV_IC) |
385 | /* | 388 | /* |
386 | * Non aliasing I-cache in HS38, | 389 | * Non aliasing I-cache in HS38, |
387 | * aliasing I-cache handled in __cache_line_loop_v3() | 390 | * aliasing I-cache handled in __cache_line_loop_v3() |
@@ -397,6 +400,55 @@ void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, | |||
397 | } | 400 | } |
398 | } | 401 | } |
399 | 402 | ||
403 | #else | ||
404 | |||
405 | /* | ||
406 | * optimized flush operation which takes a region as opposed to iterating per line | ||
407 | */ | ||
408 | static inline | ||
409 | void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, | ||
410 | unsigned long sz, const int op, const int full_page) | ||
411 | { | ||
412 | unsigned int s, e; | ||
413 | |||
414 | /* Only for Non aliasing I-cache in HS38 */ | ||
415 | if (op == OP_INV_IC) { | ||
416 | s = ARC_REG_IC_IVIR; | ||
417 | e = ARC_REG_IC_ENDR; | ||
418 | } else { | ||
419 | s = ARC_REG_DC_STARTR; | ||
420 | e = ARC_REG_DC_ENDR; | ||
421 | } | ||
422 | |||
423 | if (!full_page) { | ||
424 | /* for any leading gap between @paddr and start of cache line */ | ||
425 | sz += paddr & ~CACHE_LINE_MASK; | ||
426 | paddr &= CACHE_LINE_MASK; | ||
427 | |||
428 | /* | ||
429 | * account for any trailing gap to end of cache line | ||
430 | * this is equivalent to DIV_ROUND_UP() in line ops above | ||
431 | */ | ||
432 | sz += L1_CACHE_BYTES - 1; | ||
433 | } | ||
434 | |||
435 | if (is_pae40_enabled()) { | ||
436 | /* TBD: check if crossing 4TB boundary */ | ||
437 | if (op == OP_INV_IC) | ||
438 | write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); | ||
439 | else | ||
440 | write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); | ||
441 | } | ||
442 | |||
443 | /* ENDR needs to be set ahead of START */ | ||
444 | write_aux_reg(e, paddr + sz); /* ENDR is exclusive */ | ||
445 | write_aux_reg(s, paddr); | ||
446 | |||
447 | /* caller waits on DC_CTRL.FS */ | ||
448 | } | ||
449 | |||
450 | #endif | ||
451 | |||
400 | #if (CONFIG_ARC_MMU_VER < 3) | 452 | #if (CONFIG_ARC_MMU_VER < 3) |
401 | #define __cache_line_loop __cache_line_loop_v2 | 453 | #define __cache_line_loop __cache_line_loop_v2 |
402 | #elif (CONFIG_ARC_MMU_VER == 3) | 454 | #elif (CONFIG_ARC_MMU_VER == 3) |
@@ -411,6 +463,11 @@ void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, | |||
411 | * Machine specific helpers for Entire D-Cache or Per Line ops | 463 | * Machine specific helpers for Entire D-Cache or Per Line ops |
412 | */ | 464 | */ |
413 | 465 | ||
466 | #ifndef USE_RGN_FLSH | ||
467 | /* | ||
468 | * this version avoids extra read/write of DC_CTRL for flush or invalid ops | ||
469 | * in the non region flush regime (such as for ARCompact) | ||
470 | */ | ||
414 | static inline void __before_dc_op(const int op) | 471 | static inline void __before_dc_op(const int op) |
415 | { | 472 | { |
416 | if (op == OP_FLUSH_N_INV) { | 473 | if (op == OP_FLUSH_N_INV) { |
@@ -424,6 +481,32 @@ static inline void __before_dc_op(const int op) | |||
424 | } | 481 | } |
425 | } | 482 | } |
426 | 483 | ||
484 | #else | ||
485 | |||
486 | static inline void __before_dc_op(const int op) | ||
487 | { | ||
488 | const unsigned int ctl = ARC_REG_DC_CTRL; | ||
489 | unsigned int val = read_aux_reg(ctl); | ||
490 | |||
491 | if (op == OP_FLUSH_N_INV) { | ||
492 | val |= DC_CTRL_INV_MODE_FLUSH; | ||
493 | } | ||
494 | |||
495 | if (op != OP_INV_IC) { | ||
496 | /* | ||
497 | * Flush / Invalidate is provided by DC_CTRL.RNG_OP 0 or 1 | ||
498 | * combined Flush-n-invalidate uses DC_CTRL.IM = 1 set above | ||
499 | */ | ||
500 | val &= ~DC_CTRL_RGN_OP_MSK; | ||
501 | if (op & OP_INV) | ||
502 | val |= DC_CTRL_RGN_OP_INV; | ||
503 | } | ||
504 | write_aux_reg(ctl, val); | ||
505 | } | ||
506 | |||
507 | #endif | ||
508 | |||
509 | |||
427 | static inline void __after_dc_op(const int op) | 510 | static inline void __after_dc_op(const int op) |
428 | { | 511 | { |
429 | if (op & OP_FLUSH) { | 512 | if (op & OP_FLUSH) { |
@@ -486,13 +569,14 @@ static void __dc_enable(void) | |||
486 | static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr, | 569 | static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr, |
487 | unsigned long sz, const int op) | 570 | unsigned long sz, const int op) |
488 | { | 571 | { |
572 | const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; | ||
489 | unsigned long flags; | 573 | unsigned long flags; |
490 | 574 | ||
491 | local_irq_save(flags); | 575 | local_irq_save(flags); |
492 | 576 | ||
493 | __before_dc_op(op); | 577 | __before_dc_op(op); |
494 | 578 | ||
495 | __cache_line_loop(paddr, vaddr, sz, op); | 579 | __cache_line_loop(paddr, vaddr, sz, op, full_page); |
496 | 580 | ||
497 | __after_dc_op(op); | 581 | __after_dc_op(op); |
498 | 582 | ||
@@ -521,10 +605,11 @@ static inline void | |||
521 | __ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr, | 605 | __ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr, |
522 | unsigned long sz) | 606 | unsigned long sz) |
523 | { | 607 | { |
608 | const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; | ||
524 | unsigned long flags; | 609 | unsigned long flags; |
525 | 610 | ||
526 | local_irq_save(flags); | 611 | local_irq_save(flags); |
527 | (*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC); | 612 | (*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC, full_page); |
528 | local_irq_restore(flags); | 613 | local_irq_restore(flags); |
529 | } | 614 | } |
530 | 615 | ||