Skip to content

Commit 4a9bf88

Browse files
djbwdjbw
committed
Merge branch 'pmem-api' into libnvdimm-for-next
2 parents a06a757 + 67a3e8f commit 4a9bf88

File tree

56 files changed

+671
-315
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+671
-315
lines changed

‎Documentation/filesystems/Locking

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,8 @@ prototypes:
397397
int (*release) (struct gendisk *, fmode_t);
398398
int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
399399
int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
400-
int (*direct_access) (struct block_device *, sector_t, void **, unsigned long *);
400+
int (*direct_access) (struct block_device *, sector_t, void __pmem **,
401+
unsigned long *);
401402
int (*media_changed) (struct gendisk *);
402403
void (*unlock_native_capacity) (struct gendisk *);
403404
int (*revalidate_disk) (struct gendisk *);

‎MAINTAINERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6161,6 +6161,7 @@ Q: https://patchwork.kernel.org/project/linux-nvdimm/list/
61616161
S: Supported
61626162
F: drivers/nvdimm/pmem.c
61636163
F: include/linux/pmem.h
6164+
F: arch/*/include/asm/pmem.h
61646165

61656166
LINUX FOR IBM pSERIES (RS/6000)
61666167
M: Paul Mackerras <[email protected]>

‎arch/arm/mach-clps711x/board-cdb89712.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ static struct physmap_flash_data cdb89712_bootrom_pdata __initdata = {
9595

9696
static struct resource cdb89712_bootrom_resources[] __initdata = {
9797
DEFINE_RES_NAMED(CS7_PHYS_BASE, SZ_128, "BOOTROM", IORESOURCE_MEM |
98-
IORESOURCE_CACHEABLE | IORESOURCE_READONLY),
98+
IORESOURCE_READONLY),
9999
};
100100

101101
static struct platform_device cdb89712_bootrom_pdev __initdata = {

‎arch/arm/mach-shmobile/pm-rcar.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
#include <linux/err.h>
1313
#include <linux/mm.h>
1414
#include <linux/spinlock.h>
15-
#include <asm/io.h>
15+
#include <linux/io.h>
1616
#include "pm-rcar.h"
1717

1818
/* SYSC */

‎arch/ia64/include/asm/io.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,7 @@ static inline void __iomem * ioremap_cache (unsigned long phys_addr, unsigned lo
435435
{
436436
return ioremap(phys_addr, size);
437437
}
438+
#define ioremap_cache ioremap_cache
438439

439440

440441
/*

‎arch/ia64/kernel/cyclone.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
#include <linux/errno.h>
55
#include <linux/timex.h>
66
#include <linux/clocksource.h>
7-
#include <asm/io.h>
7+
#include <linux/io.h>
88

99
/* IBM Summit (EXA) Cyclone counter code*/
1010
#define CYCLONE_CBAR_ADDR 0xFEB00CD0

‎arch/powerpc/kernel/pci_of_scan.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ static void of_pci_parse_addrs(struct device_node *node, struct pci_dev *dev)
102102
res = &dev->resource[(i - PCI_BASE_ADDRESS_0) >> 2];
103103
} else if (i == dev->rom_base_reg) {
104104
res = &dev->resource[PCI_ROM_RESOURCE];
105-
flags |= IORESOURCE_READONLY | IORESOURCE_CACHEABLE;
105+
flags |= IORESOURCE_READONLY;
106106
} else {
107107
printk(KERN_ERR "PCI: bad cfg reg num 0x%x\n", i);
108108
continue;

‎arch/powerpc/sysdev/axonram.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -141,13 +141,14 @@ axon_ram_make_request(struct request_queue *queue, struct bio *bio)
141141
*/
142142
static long
143143
axon_ram_direct_access(struct block_device *device, sector_t sector,
144-
void **kaddr, unsigned long *pfn, long size)
144+
void __pmem **kaddr, unsigned long *pfn, long size)
145145
{
146146
struct axon_ram_bank *bank = device->bd_disk->private_data;
147147
loff_t offset = (loff_t)sector << AXON_RAM_SECTOR_SHIFT;
148+
void *addr = (void *)(bank->ph_addr + offset);
148149

149-
*kaddr = (void *)(bank->ph_addr + offset);
150-
*pfn = virt_to_phys(*kaddr) >> PAGE_SHIFT;
150+
*kaddr = (void __pmem *)addr;
151+
*pfn = virt_to_phys(addr) >> PAGE_SHIFT;
151152

152153
return bank->size - offset;
153154
}

‎arch/sh/include/asm/io.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,7 @@ ioremap_cache(phys_addr_t offset, unsigned long size)
342342
{
343343
return __ioremap_mode(offset, size, PAGE_KERNEL);
344344
}
345+
#define ioremap_cache ioremap_cache
345346

346347
#ifdef CONFIG_HAVE_IOREMAP_PROT
347348
static inline void __iomem *

‎arch/sparc/kernel/pci.c

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -231,8 +231,7 @@ static void pci_parse_of_addrs(struct platform_device *op,
231231
res = &dev->resource[(i - PCI_BASE_ADDRESS_0) >> 2];
232232
} else if (i == dev->rom_base_reg) {
233233
res = &dev->resource[PCI_ROM_RESOURCE];
234-
flags |= IORESOURCE_READONLY | IORESOURCE_CACHEABLE
235-
| IORESOURCE_SIZEALIGN;
234+
flags |= IORESOURCE_READONLY | IORESOURCE_SIZEALIGN;
236235
} else {
237236
printk(KERN_ERR "PCI: bad cfg reg num 0x%x\n", i);
238237
continue;

‎arch/x86/Kconfig

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ config X86
2828
select ARCH_HAS_FAST_MULTIPLIER
2929
select ARCH_HAS_GCOV_PROFILE_ALL
3030
select ARCH_HAS_PMEM_API
31+
select ARCH_HAS_MMIO_FLUSH
3132
select ARCH_HAS_SG_CHAIN
3233
select ARCH_HAVE_NMI_SAFE_CMPXCHG
3334
select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI

‎arch/x86/include/asm/cacheflush.h

Lines changed: 2 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,8 @@ int set_pages_rw(struct page *page, int numpages);
8989

9090
void clflush_cache_range(void *addr, unsigned int size);
9191

92+
#define mmio_flush_range(addr, size) clflush_cache_range(addr, size)
93+
9294
#ifdef CONFIG_DEBUG_RODATA
9395
void mark_rodata_ro(void);
9496
extern const int rodata_test_data;
@@ -109,75 +111,4 @@ static inline int rodata_test(void)
109111
}
110112
#endif
111113

112-
#ifdef ARCH_HAS_NOCACHE_UACCESS
113-
114-
/**
115-
* arch_memcpy_to_pmem - copy data to persistent memory
116-
* @dst: destination buffer for the copy
117-
* @src: source buffer for the copy
118-
* @n: length of the copy in bytes
119-
*
120-
* Copy data to persistent memory media via non-temporal stores so that
121-
* a subsequent arch_wmb_pmem() can flush cpu and memory controller
122-
* write buffers to guarantee durability.
123-
*/
124-
static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src,
125-
size_t n)
126-
{
127-
int unwritten;
128-
129-
/*
130-
* We are copying between two kernel buffers, if
131-
* __copy_from_user_inatomic_nocache() returns an error (page
132-
* fault) we would have already reported a general protection fault
133-
* before the WARN+BUG.
134-
*/
135-
unwritten = __copy_from_user_inatomic_nocache((void __force *) dst,
136-
(void __user *) src, n);
137-
if (WARN(unwritten, "%s: fault copying %p <- %p unwritten: %d\n",
138-
__func__, dst, src, unwritten))
139-
BUG();
140-
}
141-
142-
/**
143-
* arch_wmb_pmem - synchronize writes to persistent memory
144-
*
145-
* After a series of arch_memcpy_to_pmem() operations this drains data
146-
* from cpu write buffers and any platform (memory controller) buffers
147-
* to ensure that written data is durable on persistent memory media.
148-
*/
149-
static inline void arch_wmb_pmem(void)
150-
{
151-
/*
152-
* wmb() to 'sfence' all previous writes such that they are
153-
* architecturally visible to 'pcommit'. Note, that we've
154-
* already arranged for pmem writes to avoid the cache via
155-
* arch_memcpy_to_pmem().
156-
*/
157-
wmb();
158-
pcommit_sfence();
159-
}
160-
161-
static inline bool __arch_has_wmb_pmem(void)
162-
{
163-
#ifdef CONFIG_X86_64
164-
/*
165-
* We require that wmb() be an 'sfence', that is only guaranteed on
166-
* 64-bit builds
167-
*/
168-
return static_cpu_has(X86_FEATURE_PCOMMIT);
169-
#else
170-
return false;
171-
#endif
172-
}
173-
#else /* ARCH_HAS_NOCACHE_UACCESS i.e. ARCH=um */
174-
extern void arch_memcpy_to_pmem(void __pmem *dst, const void *src, size_t n);
175-
extern void arch_wmb_pmem(void);
176-
177-
static inline bool __arch_has_wmb_pmem(void)
178-
{
179-
return false;
180-
}
181-
#endif
182-
183114
#endif /* _ASM_X86_CACHEFLUSH_H */

‎arch/x86/include/asm/io.h

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -248,12 +248,6 @@ static inline void flush_write_buffers(void)
248248
#endif
249249
}
250250

251-
static inline void __pmem *arch_memremap_pmem(resource_size_t offset,
252-
unsigned long size)
253-
{
254-
return (void __force __pmem *) ioremap_cache(offset, size);
255-
}
256-
257251
#endif /* __KERNEL__ */
258252

259253
extern void native_io_delay(void);

‎arch/x86/include/asm/pmem.h

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
/*
2+
* Copyright(c) 2015 Intel Corporation. All rights reserved.
3+
*
4+
* This program is free software; you can redistribute it and/or modify
5+
* it under the terms of version 2 of the GNU General Public License as
6+
* published by the Free Software Foundation.
7+
*
8+
* This program is distributed in the hope that it will be useful, but
9+
* WITHOUT ANY WARRANTY; without even the implied warranty of
10+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11+
* General Public License for more details.
12+
*/
13+
#ifndef __ASM_X86_PMEM_H__
14+
#define __ASM_X86_PMEM_H__
15+
16+
#include <linux/uaccess.h>
17+
#include <asm/cacheflush.h>
18+
#include <asm/cpufeature.h>
19+
#include <asm/special_insns.h>
20+
21+
#define ARCH_MEMREMAP_PMEM MEMREMAP_WB
22+
23+
#ifdef CONFIG_ARCH_HAS_PMEM_API
24+
/**
25+
* arch_memcpy_to_pmem - copy data to persistent memory
26+
* @dst: destination buffer for the copy
27+
* @src: source buffer for the copy
28+
* @n: length of the copy in bytes
29+
*
30+
* Copy data to persistent memory media via non-temporal stores so that
31+
* a subsequent arch_wmb_pmem() can flush cpu and memory controller
32+
* write buffers to guarantee durability.
33+
*/
34+
static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src,
35+
size_t n)
36+
{
37+
int unwritten;
38+
39+
/*
40+
* We are copying between two kernel buffers, if
41+
* __copy_from_user_inatomic_nocache() returns an error (page
42+
* fault) we would have already reported a general protection fault
43+
* before the WARN+BUG.
44+
*/
45+
unwritten = __copy_from_user_inatomic_nocache((void __force *) dst,
46+
(void __user *) src, n);
47+
if (WARN(unwritten, "%s: fault copying %p <- %p unwritten: %d\n",
48+
__func__, dst, src, unwritten))
49+
BUG();
50+
}
51+
52+
/**
53+
* arch_wmb_pmem - synchronize writes to persistent memory
54+
*
55+
* After a series of arch_memcpy_to_pmem() operations this drains data
56+
* from cpu write buffers and any platform (memory controller) buffers
57+
* to ensure that written data is durable on persistent memory media.
58+
*/
59+
static inline void arch_wmb_pmem(void)
60+
{
61+
/*
62+
* wmb() to 'sfence' all previous writes such that they are
63+
* architecturally visible to 'pcommit'. Note, that we've
64+
* already arranged for pmem writes to avoid the cache via
65+
* arch_memcpy_to_pmem().
66+
*/
67+
wmb();
68+
pcommit_sfence();
69+
}
70+
71+
/**
72+
* __arch_wb_cache_pmem - write back a cache range with CLWB
73+
* @vaddr: virtual start address
74+
* @size: number of bytes to write back
75+
*
76+
* Write back a cache range using the CLWB (cache line write back)
77+
* instruction. This function requires explicit ordering with an
78+
* arch_wmb_pmem() call. This API is internal to the x86 PMEM implementation.
79+
*/
80+
static inline void __arch_wb_cache_pmem(void *vaddr, size_t size)
81+
{
82+
u16 x86_clflush_size = boot_cpu_data.x86_clflush_size;
83+
unsigned long clflush_mask = x86_clflush_size - 1;
84+
void *vend = vaddr + size;
85+
void *p;
86+
87+
for (p = (void *)((unsigned long)vaddr & ~clflush_mask);
88+
p < vend; p += x86_clflush_size)
89+
clwb(p);
90+
}
91+
92+
/*
93+
* copy_from_iter_nocache() on x86 only uses non-temporal stores for iovec
94+
* iterators, so for other types (bvec & kvec) we must do a cache write-back.
95+
*/
96+
static inline bool __iter_needs_pmem_wb(struct iov_iter *i)
97+
{
98+
return iter_is_iovec(i) == false;
99+
}
100+
101+
/**
102+
* arch_copy_from_iter_pmem - copy data from an iterator to PMEM
103+
* @addr: PMEM destination address
104+
* @bytes: number of bytes to copy
105+
* @i: iterator with source data
106+
*
107+
* Copy data from the iterator 'i' to the PMEM buffer starting at 'addr'.
108+
* This function requires explicit ordering with an arch_wmb_pmem() call.
109+
*/
110+
static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes,
111+
struct iov_iter *i)
112+
{
113+
void *vaddr = (void __force *)addr;
114+
size_t len;
115+
116+
/* TODO: skip the write-back by always using non-temporal stores */
117+
len = copy_from_iter_nocache(vaddr, bytes, i);
118+
119+
if (__iter_needs_pmem_wb(i))
120+
__arch_wb_cache_pmem(vaddr, bytes);
121+
122+
return len;
123+
}
124+
125+
/**
126+
* arch_clear_pmem - zero a PMEM memory range
127+
* @addr: virtual start address
128+
* @size: number of bytes to zero
129+
*
130+
* Write zeros into the memory range starting at 'addr' for 'size' bytes.
131+
* This function requires explicit ordering with an arch_wmb_pmem() call.
132+
*/
133+
static inline void arch_clear_pmem(void __pmem *addr, size_t size)
134+
{
135+
void *vaddr = (void __force *)addr;
136+
137+
/* TODO: implement the zeroing via non-temporal writes */
138+
if (size == PAGE_SIZE && ((unsigned long)vaddr & ~PAGE_MASK) == 0)
139+
clear_page(vaddr);
140+
else
141+
memset(vaddr, 0, size);
142+
143+
__arch_wb_cache_pmem(vaddr, size);
144+
}
145+
146+
static inline bool arch_has_wmb_pmem(void)
147+
{
148+
#ifdef CONFIG_X86_64
149+
/*
150+
* We require that wmb() be an 'sfence', that is only guaranteed on
151+
* 64-bit builds
152+
*/
153+
return static_cpu_has(X86_FEATURE_PCOMMIT);
154+
#else
155+
return false;
156+
#endif
157+
}
158+
#endif /* CONFIG_ARCH_HAS_PMEM_API */
159+
160+
#endif /* __ASM_X86_PMEM_H__ */

‎arch/xtensa/include/asm/io.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ static inline void __iomem *ioremap_cache(unsigned long offset,
5757
else
5858
BUG();
5959
}
60+
#define ioremap_cache ioremap_cache
6061

6162
#define ioremap_wc ioremap_nocache
6263
#define ioremap_wt ioremap_nocache

‎drivers/acpi/Kconfig

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,7 @@ config ACPI_NFIT
410410
tristate "ACPI NVDIMM Firmware Interface Table (NFIT)"
411411
depends on PHYS_ADDR_T_64BIT
412412
depends on BLK_DEV
413+
depends on ARCH_HAS_MMIO_FLUSH
413414
select LIBNVDIMM
414415
help
415416
Infrastructure to probe ACPI 6 compliant platforms for

0 commit comments

Comments
 (0)