1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
|
#ifndef _MCTCP_DEVICE_H
#define _MCTCP_DEVICE_H
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/version.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include "mcps_sauron.h"
enum {
MCPS_MODE_NONE = 0,
MCPS_MODE_APP_CLUSTER = 1,
MCPS_MODE_TP_THRESHOLD = 2,
MCPS_MODE_SINGLE_MIGRATE = 4,
MCPS_MODES = 4
};
enum {
// ~ NR_CPUS - 1
MCPS_CPU_ON_PENDING = NR_CPUS,
#if defined(CONFIG_MCPS_V2)
MCPS_CPU_DIRECT_GRO,
#endif // #if defined(CONFIG_MCPS_V2)
MCPS_CPU_GRO_BYPASS,
MCPS_CPU_ERROR,
};
enum {
MCPS_ARPS_LAYER = 0,
#if CONFIG_MCPS_GRO_ENABLE
MCPS_AGRO_LAYER,
#endif
MCPS_TYPE_LAYER,
};
#define VALID_UCPU(c) (c < NR_CPUS)
#define VALID_CPU(c) (0 <= c && c < NR_CPUS)
#define NR_CLUSTER 4
#define MID_CLUSTER 3
#define BIG_CLUSTER 2
#define LIT_CLUSTER 1
#define ALL_CLUSTER 0
#if defined(CONFIG_SOC_EXYNOS9830) || defined(CONFIG_SOC_EXYNOS9820)
#define CLUSTER_MAP {LIT_CLUSTER, LIT_CLUSTER, LIT_CLUSTER, LIT_CLUSTER, MID_CLUSTER, MID_CLUSTER, BIG_CLUSTER, BIG_CLUSTER}
#else
#define CLUSTER_MAP {LIT_CLUSTER, LIT_CLUSTER, LIT_CLUSTER, LIT_CLUSTER, BIG_CLUSTER, BIG_CLUSTER, BIG_CLUSTER, BIG_CLUSTER}
#endif
const static int __mcps_cpu_cluster_map[NR_CPUS] = CLUSTER_MAP;
#define CLUSTER(c) __mcps_cpu_cluster_map[(c)]
/*Declare mcps_enable .. Sync...*/
extern int mcps_enable;
extern cpumask_var_t mcps_cpu_online_mask;
#define mcps_cpu_online(cpu) cpumask_test_cpu((cpu), mcps_cpu_online_mask)
struct mcps_modes {
unsigned int mode;
struct rcu_head rcu;
};
extern struct mcps_modes __rcu *mcps_mode;
struct mcps_pantry {
struct sk_buff_head process_queue;
unsigned int received_arps;
unsigned int processed;
unsigned int enqueued;
unsigned int ignored;
unsigned int cpu;
struct mcps_pantry *ipi_list;
struct mcps_pantry *ipi_next;
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
call_single_data_t csd ____cacheline_aligned_in_smp;
#else
struct call_single_data csd ____cacheline_aligned_in_smp;
#endif
unsigned int dropped;
struct sk_buff_head input_pkt_queue;
struct napi_struct rx_napi_struct;
unsigned int gro_processed;
#if defined(CONFIG_MCPS_V2)
struct timespec gro_flush_time;
#endif // #if defined(CONFIG_MCPS_V2)
struct timespec agro_flush_time;
struct napi_struct ipi_napi_struct;
#if defined(CONFIG_MCPS_V2)
struct napi_struct gro_napi_struct; // never be touched by other core.
#endif // #if defined(CONFIG_MCPS_V2)
#if defined(CONFIG_MCPS_CHUNK_GRO)
int modem_napi_quota;
int modem_napi_work;
#endif // #if defined(CONFIG_MCPS_CHUNK_GRO)
int offline;
};
DECLARE_PER_CPU_ALIGNED(struct mcps_pantry, mcps_pantries);
DECLARE_PER_CPU_ALIGNED(struct mcps_pantry, mcps_gro_pantries);
static inline void pantry_lock(struct mcps_pantry *pantry)
{
spin_lock(&pantry->input_pkt_queue.lock);
}
static inline void pantry_unlock(struct mcps_pantry *pantry)
{
spin_unlock(&pantry->input_pkt_queue.lock);
}
static inline void pantry_ipi_lock(struct mcps_pantry *pantry)
{
spin_lock(&pantry->process_queue.lock);
}
static inline void pantry_ipi_unlock(struct mcps_pantry *pantry)
{
spin_unlock(&pantry->process_queue.lock);
}
static inline unsigned int pantry_read_enqueued(struct mcps_pantry *p)
{
unsigned int ret = 0;
pantry_lock(p);
ret = p->enqueued;
pantry_unlock(p);
return ret;
}
struct arps_meta {
struct rcu_head rcu;
struct rps_map *maps[NR_CLUSTER];
cpumask_var_t mask;
};
#define ARPS_MAP(m, t) (m->maps[t])
#define ARPS_MAP_RCU(m, t) (rcu_dereference(m)->maps[t])
extern struct arps_meta __rcu *static_arps;
extern struct arps_meta __rcu *dynamic_arps;
extern struct arps_meta __rcu *newflow_arps;
extern struct arps_meta *get_arps_rcu(void);
extern struct arps_meta *get_newflow_rcu(void);
extern struct arps_meta *get_static_arps_rcu(void);
#define NUM_FACTORS 5
#define FACTOR_QLEN 0
#define FACTOR_PROC 1
#define FACTOR_NFLO 2
#define FACTOR_DROP 3
#define FACTOR_DIST 4
struct arps_config {
struct rcu_head rcu;
unsigned int weights[NUM_FACTORS];
};
static inline void init_arps_config(struct arps_config *config)
{
int i;
for (i = 0; i < NUM_FACTORS; i++) {
config->weights[i] = 5;
}
config->weights[FACTOR_NFLO] = 2;
}
struct mcps_config {
struct sauron sauron;
struct arps_config __rcu *arps_config;
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
#else
struct notifier_block *mcps_cpu_notifier;
struct notifier_block *mcps_gro_cpu_notifier;
#endif
struct rcu_head rcu;
} ____cacheline_aligned_in_smp;
struct mcps_attribute {
struct attribute attr;
ssize_t (*show)(struct mcps_config *config,
struct mcps_attribute *attr, char *buf);
ssize_t (*store)(struct mcps_config *config,
struct mcps_attribute *attr, const char *buf, size_t len);
};
extern struct mcps_config *mcps;
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
void mcps_napi_complete(struct napi_struct *n);
int mcps_gro_cpu_startup_callback(unsigned int ocpu);
int mcps_gro_cpu_teardown_callback(unsigned int ocpu);
#else
#define mcps_napi_complete(n) __napi_complete(n)
int mcps_gro_cpu_callback(struct notifier_block *notifier, unsigned long action, void *ocpu);
#endif
int del_mcps(struct mcps_config *mcps);
int init_mcps(struct mcps_config *mcps);
static inline int smp_processor_id_safe(void)
{
int cpu = get_cpu();
put_cpu();
return cpu;
}
#if defined(CONFIG_MCPS_ICB)
void mcps_boost_clock(int cluster);
void init_mcps_icb(void);
#else
static inline void mcps_boost_clock(int cluster)
{
return;
}
static inline void init_mcps_icb(void)
{
return;
}
#endif // #if defined(CONFIG_MCPS_ICB)
#if defined(CONFIG_MCPS_ICGB) && \
!defined(CONFIG_KLAT) && \
(defined(CONFIG_SOC_EXYNOS9830) || defined(CONFIG_SOC_EXYNOS9820))
int check_mcps_in_addr(unsigned int addr);
int check_mcps_in6_addr(struct in6_addr *addr);
#else
static inline int check_mcps_in_addr(unsigned int addr)
{
return 0;
}
static inline int check_mcps_in6_addr(struct in6_addr *addr)
{
return 0;
}
#endif // #if defined(CONFIG_MCPS_ICGB) && !defined(CONFIG_KLAT) && (defined(CONFIG_SOC_EXYNOS9820) || defined(CONFIG_SOC_EXYNOS9630) || defined(CONFIG_SOC_EXYNOS9830))
#endif
|