forked from cache-ext/cache_ext
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathlinux.patch
More file actions
206 lines (188 loc) · 5.71 KB
/
linux.patch
File metadata and controls
206 lines (188 loc) · 5.71 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 3086502ac..426abad79 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1389,6 +1389,12 @@ struct cache_ext_admission_ctx {
u64 ino;
u64 offset;
u64 size;
+
+ /**
+ * 0 means invalid
+ */
+ u64 victim_ino;
+ u64 victim_page_offset;
};
// TODO: How can I make only some fields cache_ext_eviction_ctx writeable?
@@ -1400,6 +1406,7 @@ struct cache_ext_ops {
void (*folio_accessed)(struct folio *folio);
void (*folio_evicted)(struct folio *folio);
bool (*admit_folio)(struct cache_ext_admission_ctx *ctx);
+ bool (*filter_inode)(u64 ino);
// TODO: Add name?
};
diff --git a/mm/cache_ext_ds.c b/mm/cache_ext_ds.c
index cde0fca84..920f716b8 100644
--- a/mm/cache_ext_ds.c
+++ b/mm/cache_ext_ds.c
@@ -66,7 +66,7 @@ int __cache_ext_list_add_impl(struct cache_ext_list *list, struct folio *folio,
struct valid_folio *valid_folio = valid_folios_lookup(folio);
if (!valid_folio) {
spin_unlock(bucket_lock);
- return -1;
+ return -2;
}
// TODO: Make sure the cache_ext_list still exists.
@@ -79,7 +79,7 @@ int __cache_ext_list_add_impl(struct cache_ext_list *list, struct folio *folio,
if (!list_empty(&node->node)) {
cache_ext_ds_registry_write_unlock(folio);
spin_unlock(bucket_lock);
- return -1;
+ return -3;
}
if (tail)
@@ -111,7 +111,7 @@ int cache_ext_list_move(struct cache_ext_list *list, struct folio *folio,
struct valid_folio *valid_folio = valid_folios_lookup(folio);
if (!valid_folio) {
spin_unlock(bucket_lock);
- return -1;
+ return -2;
}
// Get the global list lock
diff --git a/mm/filemap.c b/mm/filemap.c
index b84763395..af27883d2 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2595,28 +2595,32 @@ static int __cache_ext_dio(struct file *file, struct address_space *mapping,
for (size_t i = index; i < last_index; ++i) {
folio = filemap_alloc_folio(mapping_gfp_mask(mapping), 0);
if (!folio) {
- if (i == index)
+ if (i == index) {
return -ENOMEM;
- // We added some folios that we want to copy and then
- // free, so this could fix our OOM
+ }
return 1;
}
filemap_invalidate_lock_shared(mapping);
__folio_set_locked(folio);
- folio_ref_add(folio, folio_nr_pages(folio));
folio->mapping = mapping;
folio->index = i;
error = filemap_read_folio_cache_ext(
file, mapping->a_ops->read_folio, folio);
filemap_invalidate_unlock_shared(mapping);
if (error) {
- if (i == index)
+ folio->mapping = NULL;
+ folio_put(folio);
+ if (i == index) {
return error;
+ }
+
return 1;
}
+ folio->mapping = NULL;
+
// Don't overflow fbatch
if (!folio_batch_add(fbatch, folio))
break;
@@ -2653,15 +2657,41 @@ static int filemap_get_pages(struct kiocb *iocb, size_t count,
memcg = mem_cgroup_from_task(current);
cache_ext_ops = get_cache_ext_ops(memcg);
+ if (cache_ext_ops && cache_ext_ops->filter_inode) {
+ if (!cache_ext_ops->filter_inode(filp->f_inode->i_ino)) {
+ // If the inode is not relevant, admit it
+ ret = false;
+ goto out_cache_ext;
+ }
+ }
+
if (cache_ext_ops && cache_ext_ops->admit_folio) {
struct cache_ext_admission_ctx ctx = {
.ino = filp->f_inode->i_ino,
.offset = iocb->ki_pos,
.size = count,
+ .victim_ino = 0,
+ .victim_page_offset = 0,
+ };
+
+ /* Get a victim candidate to compare against */
+ struct cache_ext_eviction_ctx evict_ctx = {
+ .request_nr_folios_to_evict = 1,
};
+ if (cache_ext_ops->evict_folios) {
+ cache_ext_ops->evict_folios(&evict_ctx, memcg);
+ if (evict_ctx.nr_folios_to_evict > 0) {
+ struct folio *victim = evict_ctx.folios_to_evict[0];
+ ctx.victim_ino = victim->mapping->host->i_ino;
+ ctx.victim_page_offset = victim->index;
+ }
+ }
+
ret = cache_ext_ops->admit_folio(&ctx);
}
+out_cache_ext:
+
rcu_read_unlock();
/* Admission hook decided not to add to page cache */
@@ -2767,10 +2797,12 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
break;
error = filemap_get_pages(iocb, iter->count, &fbatch, false);
- if (error < 0)
+ if (error < 0) {
break;
- else if (error == 1)
+ }
+ else if (error == 1) {
cache_ext_flag = 1;
+ }
/*
* i_size must be checked after we know the pages are Uptodate.
@@ -2797,7 +2829,8 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
*/
if (!pos_same_folio(iocb->ki_pos, last_pos - 1,
fbatch.folios[0]))
- folio_mark_accessed(fbatch.folios[0]);
+ if (!cache_ext_flag)
+ folio_mark_accessed(fbatch.folios[0]);
for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i];
@@ -2816,8 +2849,6 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
* virtual addresses, take care of potential aliasing
* before reading the folio on the kernel side.
*/
-
- // TODO: investigate
if (writably_mapped)
flush_dcache_folio(folio);
@@ -2831,15 +2862,11 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
error = -EFAULT;
break;
}
-
- // TODO: drop reference count before freeing?
- if (cache_ext_flag)
- filemap_free_folio(mapping, folio);
}
put_folios:
- if (!cache_ext_flag)
- for (i = 0; i < folio_batch_count(&fbatch); i++)
- folio_put(fbatch.folios[i]);
+ for (i = 0; i < folio_batch_count(&fbatch); i++) {
+ folio_put(fbatch.folios[i]);
+ }
cache_ext_flag = 0;
folio_batch_init(&fbatch);
@@ -3044,9 +3071,11 @@ ssize_t filemap_splice_read(struct file *in, loff_t *ppos,
break;
iocb.ki_pos = *ppos;
+
error = filemap_get_pages(&iocb, len, &fbatch, true);
- if (error < 0)
+ if (error < 0) {
break;
+ }
/*
* i_size must be checked after we know the pages are Uptodate.