summaryrefstree
path: root/kernel/drivers/soc/qcom/smcinvoke.c
blob: 3cf1246a5b007160d8eb2c0890f8ddf3a8c8c486 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2016-2020, The Linux Foundation. All rights reserved.
 */

#define pr_fmt(fmt) "smcinvoke: %s: " fmt, __func__

#include <linux/module.h>
#include <linux/mod_devicetable.h>
#include <linux/device.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/anon_inodes.h>
#include <linux/hashtable.h>
#include <linux/smcinvoke.h>
#include <linux/cdev.h>
#include <linux/uaccess.h>
#include <linux/dma-buf.h>
#include <linux/kref.h>
#include <linux/signal.h>
#include <linux/msm_ion.h>

#include <linux/qcom_scm.h>
#include <asm/cacheflush.h>
#include <soc/qcom/qseecomi.h>
#include <linux/qtee_shmbridge.h>

#include "smcinvoke_object.h"
#include "../../misc/qseecom_kernel.h"

#define SMCINVOKE_DEV                   "smcinvoke"
#define SMCINVOKE_TZ_ROOT_OBJ           1
#define SMCINVOKE_TZ_OBJ_NULL           0
#define SMCINVOKE_TZ_MIN_BUF_SIZE       4096
#define SMCINVOKE_ARGS_ALIGN_SIZE       (sizeof(uint64_t))
#define SMCINVOKE_NEXT_AVAILABLE_TXN    0
#define SMCINVOKE_REQ_PLACED            1
#define SMCINVOKE_REQ_PROCESSING        2
#define SMCINVOKE_REQ_PROCESSED         3
#define SMCINVOKE_INCREMENT             1
#define SMCINVOKE_DECREMENT             0
#define SMCINVOKE_OBJ_TYPE_TZ_OBJ       0
#define SMCINVOKE_OBJ_TYPE_SERVER       1
#define SMCINVOKE_MEM_MAP_OBJ           0
#define SMCINVOKE_MEM_RGN_OBJ           1
#define SMCINVOKE_MEM_PERM_RW           6

/* TZ defined values - Start */
#define SMCINVOKE_INVOKE_PARAM_ID       0x224
#define SMCINVOKE_CB_RSP_PARAM_ID       0x22
#define SMCINVOKE_INVOKE_CMD            0x32000602
#define SMCINVOKE_CB_RSP_CMD            0x32000601
#define SMCINVOKE_RESULT_INBOUND_REQ_NEEDED 3
/* TZ defined values - End */

/*
 * This is the state when server FD has been closed but
 * TZ still has refs of CBOBjs served by this server
 */
#define SMCINVOKE_SERVER_STATE_DEFUNCT  1

#define FOR_ARGS(ndxvar, counts, section) \
	for (ndxvar = OBJECT_COUNTS_INDEX_##section(counts); \
		ndxvar < (OBJECT_COUNTS_INDEX_##section(counts) \
		+ OBJECT_COUNTS_NUM_##section(counts)); \
		++ndxvar)

#define TZCB_BUF_OFFSET(tzcb_req) (sizeof(tzcb_req->result) + \
			sizeof(struct smcinvoke_msg_hdr) + \
			sizeof(union smcinvoke_tz_args) * \
				OBJECT_COUNTS_TOTAL(tzcb_req->hdr.counts))

/*
 * +ve uhandle : either remote obj or mem obj, decided by f_ops
 * -ve uhandle : either Obj NULL or CBObj
 *	- -1: OBJ NULL
 *	- < -1: CBObj
 */
#define UHANDLE_IS_FD(h) ((h) >= 0)
#define UHANDLE_IS_NULL(h) ((h) == SMCINVOKE_USERSPACE_OBJ_NULL)
#define UHANDLE_IS_CB_OBJ(h) (h < SMCINVOKE_USERSPACE_OBJ_NULL)
#define UHANDLE_NULL (SMCINVOKE_USERSPACE_OBJ_NULL)
/*
 * MAKE => create handle for other domain i.e. TZ or userspace
 * GET => retrieve obj from incoming handle
 */
#define UHANDLE_GET_CB_OBJ(h) (-2-(h))
#define UHANDLE_MAKE_CB_OBJ(o) (-2-(o))
#define UHANDLE_GET_FD(h) (h)

/*
 * +ve tzhandle : remote object i.e. owned by TZ
 * -ve tzhandle : local object i.e. owned by linux
 * --------------------------------------------------
 *| 1 (1 bit) | Obj Id (15 bits) | srvr id (16 bits) |
 * ---------------------------------------------------
 * Server ids are defined below for various local objects
 * server id 0 : Kernel Obj
 * server id 1 : Memory region Obj
 * server id 2 : Memory map Obj
 * server id 3-15: Reserverd
 * server id 16 & up: Callback Objs
 */
#define KRNL_SRVR_ID 0
#define MEM_RGN_SRVR_ID 1
#define MEM_MAP_SRVR_ID 2
#define CBOBJ_SERVER_ID_START 0x10
#define CBOBJ_SERVER_ID_END ((1<<16) - 1)
/* local obj id is represented by 15 bits */
#define MAX_LOCAL_OBJ_ID ((1<<15) - 1)
/* CBOBJs will be served by server id 0x10 onwards */
#define TZHANDLE_GET_SERVER(h) ((uint16_t)((h) & 0xFFFF))
#define TZHANDLE_GET_OBJID(h) (((h) >> 16) & 0x7FFF)
#define TZHANDLE_MAKE_LOCAL(s, o) (((0x8000 | (o)) << 16) | s)

#define TZHANDLE_IS_NULL(h) ((h) == SMCINVOKE_TZ_OBJ_NULL)
#define TZHANDLE_IS_LOCAL(h) ((h) & 0x80000000)
#define TZHANDLE_IS_REMOTE(h) (!TZHANDLE_IS_NULL(h) && !TZHANDLE_IS_LOCAL(h))

#define TZHANDLE_IS_KERNEL_OBJ(h) (TZHANDLE_IS_LOCAL(h) && \
				TZHANDLE_GET_SERVER(h) == KRNL_SRVR_ID)
#define TZHANDLE_IS_MEM_RGN_OBJ(h) (TZHANDLE_IS_LOCAL(h) && \
				TZHANDLE_GET_SERVER(h) == MEM_RGN_SRVR_ID)
#define TZHANDLE_IS_MEM_MAP_OBJ(h) (TZHANDLE_IS_LOCAL(h) && \
				TZHANDLE_GET_SERVER(h) == MEM_MAP_SRVR_ID)
#define TZHANDLE_IS_MEM_OBJ(h) (TZHANDLE_IS_MEM_RGN_OBJ(h) || \
				TZHANDLE_IS_MEM_MAP_OBJ(h))
#define TZHANDLE_IS_CB_OBJ(h) (TZHANDLE_IS_LOCAL(h) && \
				TZHANDLE_GET_SERVER(h) >= CBOBJ_SERVER_ID_START)

#define FILE_IS_REMOTE_OBJ(f) ((f)->f_op && (f)->f_op == &g_smcinvoke_fops)

static DEFINE_MUTEX(g_smcinvoke_lock);
#define NO_LOCK 0
#define TAKE_LOCK 1
#define MUTEX_LOCK(x) { if (x) mutex_lock(&g_smcinvoke_lock); }
#define MUTEX_UNLOCK(x) { if (x) mutex_unlock(&g_smcinvoke_lock); }
static DEFINE_HASHTABLE(g_cb_servers, 8);
static LIST_HEAD(g_mem_objs);
static uint16_t g_last_cb_server_id = CBOBJ_SERVER_ID_START;
static uint16_t g_last_mem_rgn_id, g_last_mem_map_obj_id;
static size_t g_max_cb_buf_size = SMCINVOKE_TZ_MIN_BUF_SIZE;
static unsigned int cb_reqs_inflight;

static long smcinvoke_ioctl(struct file *, unsigned int, unsigned long);
static int smcinvoke_open(struct inode *, struct file *);
static int smcinvoke_release(struct inode *, struct file *);
static int release_cb_server(uint16_t);

static const struct file_operations g_smcinvoke_fops = {
	.owner		= THIS_MODULE,
	.unlocked_ioctl	= smcinvoke_ioctl,
	.compat_ioctl	= smcinvoke_ioctl,
	.open		= smcinvoke_open,
	.release	= smcinvoke_release,
};

static dev_t smcinvoke_device_no;
static struct cdev smcinvoke_cdev;
static struct class *driver_class;
static struct device *class_dev;
static struct platform_device *smcinvoke_pdev;

struct smcinvoke_buf_hdr {
	uint32_t offset;
	uint32_t size;
};

union smcinvoke_tz_args {
	struct smcinvoke_buf_hdr b;
	int32_t handle;
};

struct smcinvoke_msg_hdr {
	uint32_t tzhandle;
	uint32_t op;
	uint32_t counts;
};

/* Inbound reqs from TZ */
struct smcinvoke_tzcb_req {
	int32_t result;
	struct smcinvoke_msg_hdr hdr;
	union smcinvoke_tz_args args[0];
};

struct smcinvoke_file_data {
	uint32_t context_type;
	union {
		uint32_t tzhandle;
		uint16_t server_id;
	};
};

struct smcinvoke_piggyback_msg {
	uint32_t version;
	uint32_t op;
	uint32_t counts;
	int32_t objs[0];
};

/* Data structure to hold request coming from TZ */
struct smcinvoke_cb_txn {
	uint32_t txn_id;
	int32_t state;
	struct smcinvoke_tzcb_req *cb_req;
	size_t cb_req_bytes;
	struct file **filp_to_release;
	struct hlist_node hash;
	struct kref ref_cnt;
};

struct smcinvoke_server_info {
	uint16_t server_id;
	uint16_t state;
	uint32_t txn_id;
	struct kref ref_cnt;
	wait_queue_head_t req_wait_q;
	wait_queue_head_t rsp_wait_q;
	size_t cb_buf_size;
	DECLARE_HASHTABLE(reqs_table, 4);
	DECLARE_HASHTABLE(responses_table, 4);
	struct hlist_node hash;
	struct list_head pending_cbobjs;
};

struct smcinvoke_cbobj {
	uint16_t cbobj_id;
	struct kref ref_cnt;
	struct smcinvoke_server_info *server;
	struct list_head list;
};

/*
 * We require couple of objects, one for mem region & another
 * for mapped mem_obj once mem region has been mapped. It is
 * possible that TZ can release either independent of other.
 */
struct smcinvoke_mem_obj {
	/* these ids are objid part of tzhandle */
	uint16_t mem_region_id;
	uint16_t mem_map_obj_id;
	struct dma_buf *dma_buf;
	struct dma_buf_attachment *buf_attach;
	struct sg_table *sgt;
	struct kref mem_regn_ref_cnt;
	struct kref mem_map_obj_ref_cnt;
	uint64_t p_addr;
	size_t p_addr_len;
	struct list_head list;
	bool bridge_created_by_others;
	uint64_t shmbridge_handle;
};

static void destroy_cb_server(struct kref *kref)
{
	struct smcinvoke_server_info *server = container_of(kref,
					struct smcinvoke_server_info, ref_cnt);
	if (server) {
		hash_del(&server->hash);
		kfree(server);
	}
}

/*
 *  A separate find func is reqd mainly for couple of cases:
 *  next_cb_server_id_locked which checks if server id had been utilized or not.
 *      - It would be overhead if we do ref_cnt for this case
 *  smcinvoke_release: which is called when server is closed from userspace.
 *      - During server creation we init ref count, now put it back
 */
static struct smcinvoke_server_info *find_cb_server_locked(uint16_t server_id)
{
	struct smcinvoke_server_info *data = NULL;

	hash_for_each_possible(g_cb_servers, data, hash, server_id) {
		if (data->server_id == server_id)
			return data;
	}
	return  NULL;
}

static struct smcinvoke_server_info *get_cb_server_locked(uint16_t server_id)
{
	struct smcinvoke_server_info *server = find_cb_server_locked(server_id);

	if (server)
		kref_get(&server->ref_cnt);

	return server;
}

static uint16_t next_cb_server_id_locked(void)
{
	if (g_last_cb_server_id == CBOBJ_SERVER_ID_END)
		g_last_cb_server_id = CBOBJ_SERVER_ID_START;

	while (find_cb_server_locked(++g_last_cb_server_id))
		;

	return g_last_cb_server_id;
}

static inline void release_filp(struct file **filp_to_release, size_t arr_len)
{
	size_t i = 0;

	for (i = 0; i < arr_len; i++) {
		if (filp_to_release[i]) {
			fput(filp_to_release[i]);
			filp_to_release[i] = NULL;
		}
	}
}

static  struct smcinvoke_mem_obj *find_mem_obj_locked(uint16_t mem_obj_id,
							bool is_mem_rgn_obj)
{
	struct smcinvoke_mem_obj *mem_obj = NULL;

	if (list_empty(&g_mem_objs))
		return NULL;

	list_for_each_entry(mem_obj, &g_mem_objs, list) {
		if ((is_mem_rgn_obj &&
			(mem_obj->mem_region_id == mem_obj_id)) ||
		    (!is_mem_rgn_obj &&
			(mem_obj->mem_map_obj_id == mem_obj_id)))
			return mem_obj;
	}
	return NULL;
}

static uint32_t next_mem_region_obj_id_locked(void)
{
	if (g_last_mem_rgn_id == MAX_LOCAL_OBJ_ID)
		g_last_mem_rgn_id = 0;

	while (find_mem_obj_locked(++g_last_mem_rgn_id, SMCINVOKE_MEM_RGN_OBJ))
		;

	return g_last_mem_rgn_id;
}

static uint32_t next_mem_map_obj_id_locked(void)
{
	if (g_last_mem_map_obj_id == MAX_LOCAL_OBJ_ID)
		g_last_mem_map_obj_id = 0;

	while (find_mem_obj_locked(++g_last_mem_map_obj_id,
					SMCINVOKE_MEM_MAP_OBJ))
		;

	return g_last_mem_map_obj_id;
}

static inline void free_mem_obj_locked(struct smcinvoke_mem_obj *mem_obj)
{
	list_del(&mem_obj->list);
	dma_buf_put(mem_obj->dma_buf);
	if (!mem_obj->bridge_created_by_others)
		qtee_shmbridge_deregister(mem_obj->shmbridge_handle);
	kfree(mem_obj);
}

static void del_mem_regn_obj_locked(struct kref *kref)
{
	struct smcinvoke_mem_obj *mem_obj = container_of(kref,
			struct smcinvoke_mem_obj, mem_regn_ref_cnt);

	/*
	 * mem_regn obj and mem_map obj are held into mem_obj structure which
	 * can't be released until both kinds of objs have been released.
	 * So check whether mem_map iobj has ref 0 and only then release mem_obj
	 */
	if (kref_read(&mem_obj->mem_map_obj_ref_cnt) == 0)
		free_mem_obj_locked(mem_obj);
}

static void del_mem_map_obj_locked(struct kref *kref)
{
	struct smcinvoke_mem_obj *mem_obj = container_of(kref,
			struct smcinvoke_mem_obj, mem_map_obj_ref_cnt);

	mem_obj->p_addr_len = 0;
	mem_obj->p_addr = 0;
	if (mem_obj->sgt)
		dma_buf_unmap_attachment(mem_obj->buf_attach,
					mem_obj->sgt, DMA_BIDIRECTIONAL);
	if (mem_obj->buf_attach)
		dma_buf_detach(mem_obj->dma_buf, mem_obj->buf_attach);

	/*
	 * mem_regn obj and mem_map obj are held into mem_obj structure which
	 * can't be released until both kinds of objs have been released.
	 * So check if mem_regn obj has ref 0 and only then release mem_obj
	 */
	if (kref_read(&mem_obj->mem_regn_ref_cnt) == 0)
		free_mem_obj_locked(mem_obj);
}

static int release_mem_obj_locked(int32_t tzhandle)
{
	int is_mem_regn_obj = TZHANDLE_IS_MEM_RGN_OBJ(tzhandle);
	struct smcinvoke_mem_obj *mem_obj = find_mem_obj_locked(
			TZHANDLE_GET_OBJID(tzhandle), is_mem_regn_obj);

	if (!mem_obj) {
		pr_err("memory object not found\n");
		return OBJECT_ERROR_BADOBJ;
	}

	if (is_mem_regn_obj)
		kref_put(&mem_obj->mem_regn_ref_cnt, del_mem_regn_obj_locked);
	else
		kref_put(&mem_obj->mem_map_obj_ref_cnt, del_mem_map_obj_locked);

	return OBJECT_OK;
}

static void free_pending_cbobj_locked(struct kref *kref)
{
	struct smcinvoke_server_info *server = NULL;
	struct smcinvoke_cbobj *obj = container_of(kref,
					struct smcinvoke_cbobj, ref_cnt);
	list_del(&obj->list);
	server = obj->server;
	kfree(obj);
	if (server)
		kref_put(&server->ref_cnt, destroy_cb_server);
}

static int get_pending_cbobj_locked(uint16_t srvr_id, int16_t obj_id)
{
	int ret = 0;
	bool release_server = true;
	struct list_head *head = NULL;
	struct smcinvoke_cbobj *cbobj = NULL;
	struct smcinvoke_cbobj *obj = NULL;
	struct smcinvoke_server_info *server = get_cb_server_locked(srvr_id);

	if (!server) {
		pr_err("%s, server id : %u not found\n", __func__, srvr_id);
		return OBJECT_ERROR_BADOBJ;
	}

	head = &server->pending_cbobjs;
	list_for_each_entry(cbobj, head, list)
		if (cbobj->cbobj_id == obj_id)  {
			kref_get(&cbobj->ref_cnt);
			goto out;
		}

	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
	if (!obj) {
		ret = OBJECT_ERROR_KMEM;
		goto out;
	}

	obj->cbobj_id = obj_id;
	kref_init(&obj->ref_cnt);
	obj->server = server;
	/*
	 * we are holding server ref in cbobj; we will
	 * release server ref when cbobj is destroyed
	 */
	release_server = false;
	list_add_tail(&obj->list, head);
out:
	if (release_server)
		kref_put(&server->ref_cnt, destroy_cb_server);
	return ret;
}

static int put_pending_cbobj_locked(uint16_t srvr_id, int16_t obj_id)
{
	int ret = -EINVAL;
	struct smcinvoke_server_info *srvr_info =
					get_cb_server_locked(srvr_id);
	struct list_head *head = NULL;
	struct smcinvoke_cbobj *cbobj = NULL;

	if (!srvr_info) {
		pr_err("%s, server id : %u not found\n", __func__, srvr_id);
		return ret;
	}

	head = &srvr_info->pending_cbobjs;
	list_for_each_entry(cbobj, head, list)
		if (cbobj->cbobj_id == obj_id)  {
			kref_put(&cbobj->ref_cnt, free_pending_cbobj_locked);
			ret = 0;
			break;
		}
	kref_put(&srvr_info->ref_cnt, destroy_cb_server);
	return ret;
}

static int release_tzhandle_locked(int32_t tzhandle)
{
	if (TZHANDLE_IS_MEM_OBJ(tzhandle))
		return release_mem_obj_locked(tzhandle);
	else if (TZHANDLE_IS_CB_OBJ(tzhandle))
		return put_pending_cbobj_locked(TZHANDLE_GET_SERVER(tzhandle),
						TZHANDLE_GET_OBJID(tzhandle));
	return OBJECT_ERROR;
}

static void release_tzhandles(const int32_t *tzhandles, size_t len)
{
	size_t i;

	mutex_lock(&g_smcinvoke_lock);
	for (i = 0; i < len; i++)
		release_tzhandle_locked(tzhandles[i]);
	mutex_unlock(&g_smcinvoke_lock);
}

static void delete_cb_txn(struct kref *kref)
{
	struct smcinvoke_cb_txn *cb_txn = container_of(kref,
					struct smcinvoke_cb_txn, ref_cnt);

	if (OBJECT_OP_METHODID(cb_txn->cb_req->hdr.op) == OBJECT_OP_RELEASE)
		release_tzhandle_locked(cb_txn->cb_req->hdr.tzhandle);

	kfree(cb_txn->cb_req);
	hash_del(&cb_txn->hash);
	kfree(cb_txn);
}

static struct smcinvoke_cb_txn *find_cbtxn_locked(
				struct smcinvoke_server_info *server,
				uint32_t txn_id, int32_t state)
{
	int i = 0;
	struct smcinvoke_cb_txn *cb_txn = NULL;

	/*
	 * Since HASH_BITS() does not work on pointers, we can't select hash
	 * table using state and loop over it.
	 */
	if (state == SMCINVOKE_REQ_PLACED) {
		/* pick up 1st req */
		hash_for_each(server->reqs_table, i, cb_txn, hash) {
			kref_get(&cb_txn->ref_cnt);
			hash_del(&cb_txn->hash);
			return cb_txn;
		}
	} else if (state == SMCINVOKE_REQ_PROCESSING) {
		hash_for_each_possible(
				server->responses_table, cb_txn, hash, txn_id) {
			if (cb_txn->txn_id == txn_id) {
				kref_get(&cb_txn->ref_cnt);
				hash_del(&cb_txn->hash);
				return cb_txn;
			}
		}
	}
	return NULL;
}

/*
 * size_add saturates at SIZE_MAX. If integer overflow is detected,
 * this function would return SIZE_MAX otherwise normal a+b is returned.
 */
static inline size_t size_add(size_t a, size_t b)
{
	return (b > (SIZE_MAX - a)) ? SIZE_MAX : a + b;
}

/*
 * pad_size is used along with size_align to define a buffer overflow
 * protected version of ALIGN
 */
static inline size_t pad_size(size_t a, size_t b)
{
	return (~a + 1) % b;
}

/*
 * size_align saturates at SIZE_MAX. If integer overflow is detected, this
 * function would return SIZE_MAX otherwise next aligned size is returned.
 */
static inline size_t size_align(size_t a, size_t b)
{
	return size_add(a, pad_size(a, b));
}

static uint16_t get_server_id(int cb_server_fd)
{
	uint16_t server_id = 0;
	struct smcinvoke_file_data *svr_cxt = NULL;
	struct file *tmp_filp = fget(cb_server_fd);

	if (!tmp_filp)
		return server_id;

	svr_cxt = tmp_filp->private_data;
	if (svr_cxt && svr_cxt->context_type ==  SMCINVOKE_OBJ_TYPE_SERVER)
		server_id = svr_cxt->server_id;

	if (tmp_filp)
		fput(tmp_filp);

	return server_id;
}

static bool is_dma_fd(int32_t uhandle, struct dma_buf **dma_buf)
{
	*dma_buf = dma_buf_get(uhandle);
	return IS_ERR_OR_NULL(*dma_buf) ? false : true;
}

static bool is_remote_obj(int32_t uhandle, struct smcinvoke_file_data **tzobj,
							struct file **filp)
{
	bool ret = false;
	struct file *tmp_filp = fget(uhandle);

	if (!tmp_filp)
		return ret;

	if (FILE_IS_REMOTE_OBJ(tmp_filp)) {
		*tzobj = tmp_filp->private_data;
		if ((*tzobj)->context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ) {
			*filp = tmp_filp;
			tmp_filp = NULL;
			ret = true;
		}
	}

	if (tmp_filp)
		fput(tmp_filp);
	return ret;
}

static int create_mem_obj(struct dma_buf *dma_buf, int32_t *mem_obj)
{
	struct smcinvoke_mem_obj *t_mem_obj =
				kzalloc(sizeof(*t_mem_obj), GFP_KERNEL);

	if (!t_mem_obj) {
		dma_buf_put(dma_buf);
		return -ENOMEM;
	}

	kref_init(&t_mem_obj->mem_regn_ref_cnt);
	t_mem_obj->dma_buf = dma_buf;
	mutex_lock(&g_smcinvoke_lock);
	t_mem_obj->mem_region_id = next_mem_region_obj_id_locked();
	list_add_tail(&t_mem_obj->list, &g_mem_objs);
	mutex_unlock(&g_smcinvoke_lock);
	*mem_obj = TZHANDLE_MAKE_LOCAL(MEM_RGN_SRVR_ID,
					t_mem_obj->mem_region_id);
	return 0;
}

/*
 * This function retrieves file pointer corresponding to FD provided. It stores
 * retrieved file pointer until IOCTL call is concluded. Once call is completed,
 * all stored file pointers are released. file pointers are stored to prevent
 * other threads from releasing that FD while IOCTL is in progress.
 */
static int get_tzhandle_from_uhandle(int32_t uhandle, int32_t server_fd,
				struct file **filp, uint32_t *tzhandle)
{
	int ret = -EBADF;
	uint16_t server_id = 0;

	if (UHANDLE_IS_NULL(uhandle)) {
		*tzhandle = SMCINVOKE_TZ_OBJ_NULL;
		ret = 0;
	} else if (UHANDLE_IS_CB_OBJ(uhandle)) {
		server_id = get_server_id(server_fd);
		if (server_id < CBOBJ_SERVER_ID_START)
			goto out;

		mutex_lock(&g_smcinvoke_lock);
		ret = get_pending_cbobj_locked(server_id,
					UHANDLE_GET_CB_OBJ(uhandle));
		mutex_unlock(&g_smcinvoke_lock);
		if (ret)
			goto out;
		*tzhandle = TZHANDLE_MAKE_LOCAL(server_id,
						UHANDLE_GET_CB_OBJ(uhandle));
		ret = 0;
	} else if (UHANDLE_IS_FD(uhandle)) {
		struct dma_buf *dma_buf = NULL;
		struct smcinvoke_file_data *tzobj = NULL;

		if (is_dma_fd(UHANDLE_GET_FD(uhandle), &dma_buf)) {
			ret = create_mem_obj(dma_buf, tzhandle);
		} else if (is_remote_obj(UHANDLE_GET_FD(uhandle),
						&tzobj, filp)) {
			*tzhandle = tzobj->tzhandle;
			ret = 0;
		}
	}
out:
	return ret;
}

static int get_fd_for_obj(uint32_t obj_type, uint32_t obj, int32_t *fd)
{
	int unused_fd = -1, ret = -EINVAL;
	struct file *f = NULL;
	struct smcinvoke_file_data *cxt = NULL;

	cxt = kzalloc(sizeof(*cxt), GFP_KERNEL);
	if (!cxt) {
		ret = -ENOMEM;
		goto out;
	}
	if (obj_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ) {
		cxt->context_type = SMCINVOKE_OBJ_TYPE_TZ_OBJ;
		cxt->tzhandle = obj;
	} else if (obj_type == SMCINVOKE_OBJ_TYPE_SERVER) {
		cxt->context_type = SMCINVOKE_OBJ_TYPE_SERVER;
		cxt->server_id = obj;
	} else {
		goto out;
	}

	unused_fd = get_unused_fd_flags(O_RDWR);
	if (unused_fd < 0)
		goto out;

	if (fd == NULL)
		goto out;

	f = anon_inode_getfile(SMCINVOKE_DEV, &g_smcinvoke_fops, cxt, O_RDWR);
	if (IS_ERR(f))
		goto out;

	*fd = unused_fd;
	fd_install(*fd, f);
	return 0;
out:
	if (unused_fd >= 0)
		put_unused_fd(unused_fd);
	kfree(cxt);

	return ret;
}

static int get_uhandle_from_tzhandle(int32_t tzhandle, int32_t srvr_id,
				int32_t *uhandle, bool lock)
{
	int ret = -1;

	if (TZHANDLE_IS_NULL(tzhandle)) {
		*uhandle = UHANDLE_NULL;
		ret = 0;
	} else if (TZHANDLE_IS_CB_OBJ(tzhandle)) {
		if (srvr_id != TZHANDLE_GET_SERVER(tzhandle))
			goto out;
		*uhandle = UHANDLE_MAKE_CB_OBJ(TZHANDLE_GET_OBJID(tzhandle));
		MUTEX_LOCK(lock)
		ret = get_pending_cbobj_locked(TZHANDLE_GET_SERVER(tzhandle),
						TZHANDLE_GET_OBJID(tzhandle));
		MUTEX_UNLOCK(lock)
	} else if (TZHANDLE_IS_MEM_RGN_OBJ(tzhandle)) {
		struct smcinvoke_mem_obj *mem_obj =  NULL;

		MUTEX_LOCK(lock)
		mem_obj = find_mem_obj_locked(TZHANDLE_GET_OBJID(tzhandle),
						SMCINVOKE_MEM_RGN_OBJ);

		if (mem_obj != NULL) {
			unsigned long flags = 0;
			int fd;

			if (dma_buf_get_flags(mem_obj->dma_buf, &flags))
				goto exit_lock;
			fd = dma_buf_fd(mem_obj->dma_buf, flags);

			if (fd < 0)
				goto exit_lock;
			*uhandle = fd;
			ret = 0;
		}
exit_lock:
		MUTEX_UNLOCK(lock)
	} else if (TZHANDLE_IS_REMOTE(tzhandle)) {
		/* if execution comes here => tzhandle is an unsigned int */
		ret = get_fd_for_obj(SMCINVOKE_OBJ_TYPE_TZ_OBJ,
						(uint32_t)tzhandle, uhandle);
	}
out:
	return ret;
}

static int smcinvoke_create_bridge(struct smcinvoke_mem_obj *mem_obj)
{
	int ret = 0, i;
	int tz_perm = PERM_READ|PERM_WRITE;
	uint32_t *vmid_list;
	uint32_t *perms_list;
	uint32_t nelems = 0;
	unsigned long dma_buf_flags = 0;
	struct dma_buf *dmabuf = mem_obj->dma_buf;
	phys_addr_t phys = mem_obj->p_addr;
	size_t size = mem_obj->p_addr_len;

	if (!qtee_shmbridge_is_enabled())
		return 0;

	ret = dma_buf_get_flags(dmabuf, &dma_buf_flags);
	if (ret) {
		pr_err("failed to get dmabuf flag for mem_region_id %d\n",
				mem_obj->mem_region_id);
		return ret;
	}

	if (dma_buf_flags & ION_FLAG_SECURE)
		nelems = ion_get_flags_num_vm_elems(dma_buf_flags);
	else
		nelems = 1;

	vmid_list = kcalloc(nelems, sizeof(*vmid_list), GFP_KERNEL);
	if (!vmid_list) {
		ret = -ENOMEM;
		goto exit;
	}

	perms_list = kcalloc(nelems, sizeof(*perms_list), GFP_KERNEL);
	if (!perms_list) {
		ret = -ENOMEM;
		goto exit_free_vmid_list;
	}

	if (dma_buf_flags & ION_FLAG_SECURE) {
		ret = ion_populate_vm_list(dma_buf_flags, vmid_list, nelems);
		if (ret)
			goto exit_free_vmid_list;

		for (i = 0; i < nelems; i++)
			perms_list[i] = msm_secure_get_vmid_perms(vmid_list[i]);
	} else {
		vmid_list[0] = VMID_HLOS;
		perms_list[0] = PERM_READ | PERM_WRITE;
	}

	ret = qtee_shmbridge_register(phys, size, vmid_list, perms_list, nelems,
				      tz_perm, &mem_obj->shmbridge_handle);

	if (ret && ret != -EEXIST) {
		pr_err("creation of shm bridge for mem_region_id %d failed ret %d\n",
		       mem_obj->mem_region_id, ret);
		goto exit_free_perms_list;
	}

	if (ret == -EEXIST) {
		mem_obj->bridge_created_by_others = true;
		ret = 0;
	}

	pr_debug("created shm bridge handle %lld for mem_region_id %d\n",
			mem_obj->shmbridge_handle, mem_obj->mem_region_id);

exit_free_perms_list:
	kfree(perms_list);
exit_free_vmid_list:
	kfree(vmid_list);
exit:
	return ret;
}

static int32_t smcinvoke_release_mem_obj_locked(void *buf, size_t buf_len)
{
	struct smcinvoke_tzcb_req *msg = buf;

	if (msg->hdr.counts != OBJECT_COUNTS_PACK(0, 0, 0, 0)) {
		pr_err("Invalid object count in %s\n", __func__);
		return OBJECT_ERROR_INVALID;
	}

	return release_tzhandle_locked(msg->hdr.tzhandle);
}

static int32_t smcinvoke_map_mem_region(void *buf, size_t buf_len)
{
	int ret = OBJECT_OK;
	struct smcinvoke_tzcb_req *msg = buf;
	struct {
		uint64_t p_addr;
		uint64_t len;
		uint32_t perms;
	} *ob = NULL;
	int32_t *oo = NULL;
	struct smcinvoke_mem_obj *mem_obj = NULL;
	struct dma_buf_attachment *buf_attach = NULL;
	struct sg_table *sgt = NULL;

	if (msg->hdr.counts != OBJECT_COUNTS_PACK(0, 1, 1, 1) ||
		(buf_len - msg->args[0].b.offset <  msg->args[0].b.size)) {
		pr_err("Invalid counts received for mapping mem obj\n");
		return OBJECT_ERROR_INVALID;
	}
	/* args[0] = BO, args[1] = OI, args[2] = OO */
	ob = buf + msg->args[0].b.offset;
	oo =  &msg->args[2].handle;

	mutex_lock(&g_smcinvoke_lock);
	mem_obj = find_mem_obj_locked(TZHANDLE_GET_OBJID(msg->args[1].handle),
						SMCINVOKE_MEM_RGN_OBJ);
	if (!mem_obj) {
		mutex_unlock(&g_smcinvoke_lock);
		pr_err("Memory object not found\n");
		return OBJECT_ERROR_BADOBJ;
	}

	if (!mem_obj->p_addr) {
		kref_init(&mem_obj->mem_map_obj_ref_cnt);
		buf_attach = dma_buf_attach(mem_obj->dma_buf,
					&smcinvoke_pdev->dev);
		if (IS_ERR(buf_attach)) {
			ret = OBJECT_ERROR_KMEM;
			pr_err("dma buf attach failed, ret: %d\n", ret);
			goto out;
		}
		mem_obj->buf_attach = buf_attach;

		sgt = dma_buf_map_attachment(buf_attach, DMA_BIDIRECTIONAL);
		if (IS_ERR(sgt)) {
			pr_err("mapping dma buffers failed, ret: %d\n",
					PTR_ERR(sgt));
			ret = OBJECT_ERROR_KMEM;
			goto out;
		}
		mem_obj->sgt = sgt;

		/* contiguous only => nents=1 */
		if (sgt->nents != 1) {
			ret = OBJECT_ERROR_INVALID;
			pr_err("sg enries are not contigous, ret: %d\n", ret);
			goto out;
		}
		mem_obj->p_addr = sg_dma_address(sgt->sgl);
		mem_obj->p_addr_len = sgt->sgl->length;
		if (!mem_obj->p_addr) {
			ret = OBJECT_ERROR_INVALID;
			pr_err("invalid physical address, ret: %d\n", ret);
			goto out;
		}
		ret = smcinvoke_create_bridge(mem_obj);
		if (ret) {
			ret = OBJECT_ERROR_INVALID;
			goto out;
		}
		mem_obj->mem_map_obj_id = next_mem_map_obj_id_locked();
	} else {
		kref_get(&mem_obj->mem_map_obj_ref_cnt);
	}
	ob->p_addr = mem_obj->p_addr;
	ob->len = mem_obj->p_addr_len;
	ob->perms = SMCINVOKE_MEM_PERM_RW;
	*oo = TZHANDLE_MAKE_LOCAL(MEM_MAP_SRVR_ID, mem_obj->mem_map_obj_id);
out:
	if (ret != OBJECT_OK)
		kref_put(&mem_obj->mem_map_obj_ref_cnt, del_mem_map_obj_locked);
	mutex_unlock(&g_smcinvoke_lock);
	return ret;
}

static void process_kernel_obj(void *buf, size_t buf_len)
{
	struct smcinvoke_tzcb_req *cb_req = buf;

	switch (cb_req->hdr.op) {
	case OBJECT_OP_MAP_REGION:
		cb_req->result = smcinvoke_map_mem_region(buf, buf_len);
		break;
	case OBJECT_OP_YIELD:
		cb_req->result = OBJECT_OK;
		break;
	default:
		pr_err(" invalid operation for tz kernel object\n");
		cb_req->result = OBJECT_ERROR_INVALID;
		break;
	}
}

static void process_mem_obj(void *buf, size_t buf_len)
{
	struct smcinvoke_tzcb_req *cb_req = buf;

	mutex_lock(&g_smcinvoke_lock);
	cb_req->result = (cb_req->hdr.op == OBJECT_OP_RELEASE) ?
			smcinvoke_release_mem_obj_locked(buf, buf_len) :
			OBJECT_ERROR_INVALID;
	mutex_unlock(&g_smcinvoke_lock);
}

/*
 * Buf should be aligned to struct smcinvoke_tzcb_req
 */
static void process_tzcb_req(void *buf, size_t buf_len, struct file **arr_filp)
{
	/* ret is going to TZ. Provide values from OBJECT_ERROR_<> */
	int ret = OBJECT_ERROR_DEFUNCT;
	struct smcinvoke_cb_txn *cb_txn = NULL;
	struct smcinvoke_tzcb_req *cb_req = NULL, *tmp_cb_req = NULL;
	struct smcinvoke_server_info *srvr_info = NULL;

	if (buf_len < sizeof(struct smcinvoke_tzcb_req)) {
		pr_err("smaller buffer length : %u\n", buf_len);
		return;
	}

	cb_req = buf;

	/* check whether it is to be served by kernel or userspace */
	if (TZHANDLE_IS_KERNEL_OBJ(cb_req->hdr.tzhandle)) {
		return process_kernel_obj(buf, buf_len);
	} else if (TZHANDLE_IS_MEM_OBJ(cb_req->hdr.tzhandle)) {
		return process_mem_obj(buf, buf_len);
	} else if (!TZHANDLE_IS_CB_OBJ(cb_req->hdr.tzhandle)) {
		pr_err("Request object is not a callback object\n");
		cb_req->result = OBJECT_ERROR_INVALID;
		return;
	}

	/*
	 * We need a copy of req that could be sent to server. Otherwise, if
	 * someone kills invoke caller, buf would go away and server would be
	 * working on already freed buffer, causing a device crash.
	 */
	tmp_cb_req = kmemdup(buf, buf_len, GFP_KERNEL);
	if (!tmp_cb_req) {
		/* we need to return error to caller so fill up result */
		cb_req->result = OBJECT_ERROR_KMEM;
		pr_err("failed to create copy of request, set result: %d\n",
							cb_req->result);
		return;
	}

	cb_txn = kzalloc(sizeof(*cb_txn), GFP_KERNEL);
	if (!cb_txn) {
		cb_req->result = OBJECT_ERROR_KMEM;
		pr_err("failed to allocate memory for request, result: %d\n",
							cb_req->result);
		kfree(tmp_cb_req);
		return;
	}
	/* no need for memcpy as we did kmemdup() above */
	cb_req  = tmp_cb_req;

	cb_txn->state = SMCINVOKE_REQ_PLACED;
	cb_txn->cb_req = cb_req;
	cb_txn->cb_req_bytes = buf_len;
	cb_txn->filp_to_release = arr_filp;
	kref_init(&cb_txn->ref_cnt);

	mutex_lock(&g_smcinvoke_lock);
	++cb_reqs_inflight;
	srvr_info = get_cb_server_locked(
				TZHANDLE_GET_SERVER(cb_req->hdr.tzhandle));
	if (!srvr_info || srvr_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT) {
		/* ret equals Object_ERROR_DEFUNCT, at this point go to out */
		if (!srvr_info)
			pr_err("server is invalid\n");
		else {
			pr_err("server is defunct, state= %d tzhandle = %d\n",
				srvr_info->state, cb_req->hdr.tzhandle);
		}
		mutex_unlock(&g_smcinvoke_lock);
		goto out;
	}

	cb_txn->txn_id = ++srvr_info->txn_id;
	hash_add(srvr_info->reqs_table, &cb_txn->hash, cb_txn->txn_id);
	mutex_unlock(&g_smcinvoke_lock);
	/*
	 * we need not worry that server_info will be deleted because as long
	 * as this CBObj is served by this server, srvr_info will be valid.
	 */
	wake_up_interruptible_all(&srvr_info->req_wait_q);
	ret = wait_event_interruptible(srvr_info->rsp_wait_q,
		(cb_txn->state == SMCINVOKE_REQ_PROCESSED) ||
		(srvr_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT));

out:
	/*
	 * we could be here because of either: a. Req is PROCESSED
	 * b. Server was killed                c. Invoke thread is killed
	 * sometime invoke thread and server are part of same process.
	 */
	mutex_lock(&g_smcinvoke_lock);
	hash_del(&cb_txn->hash);
	if (cb_txn->state == SMCINVOKE_REQ_PROCESSED) {
		/*
		 * it is possible that server was killed immediately
		 * after CB Req was processed but who cares now!
		 */
	} else if (!srvr_info ||
		srvr_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT) {
		cb_req->result = OBJECT_ERROR_DEFUNCT;
		pr_err("server invalid, res: %d\n", cb_req->result);
	} else {
		pr_debug("%s wait_event interrupted ret = %d\n", __func__, ret);
		cb_req->result = OBJECT_ERROR_ABORT;
	}
	--cb_reqs_inflight;
	memcpy(buf, cb_req, buf_len);
	kref_put(&cb_txn->ref_cnt, delete_cb_txn);
	if (srvr_info)
		kref_put(&srvr_info->ref_cnt, destroy_cb_server);
	mutex_unlock(&g_smcinvoke_lock);
}

static int marshal_out_invoke_req(const uint8_t *buf, uint32_t buf_size,
				struct smcinvoke_cmd_req *req,
				union smcinvoke_arg *args_buf)
{
	int ret = -EINVAL, i = 0;
	union smcinvoke_tz_args *tz_args = NULL;
	size_t offset = sizeof(struct smcinvoke_msg_hdr) +
				OBJECT_COUNTS_TOTAL(req->counts) *
					sizeof(union smcinvoke_tz_args);

	if (offset > buf_size)
		goto out;

	tz_args = (union smcinvoke_tz_args *)
				(buf + sizeof(struct smcinvoke_msg_hdr));

	tz_args += OBJECT_COUNTS_NUM_BI(req->counts);

	if (args_buf == NULL)
		return 0;

	FOR_ARGS(i, req->counts, BO) {
		args_buf[i].b.size = tz_args->b.size;
		if ((buf_size - tz_args->b.offset < tz_args->b.size) ||
			tz_args->b.offset > buf_size) {
			pr_err("%s: buffer overflow detected\n", __func__);
			goto out;
		}
		if (copy_to_user((void __user *)(uintptr_t)(args_buf[i].b.addr),
			(uint8_t *)(buf) + tz_args->b.offset,
						tz_args->b.size)) {
			pr_err("Error %d copying ctxt to user\n", ret);
			goto out;
		}
		tz_args++;
	}
	tz_args += OBJECT_COUNTS_NUM_OI(req->counts);

	FOR_ARGS(i, req->counts, OO) {
		/*
		 * create a new FD and assign to output object's context.
		 * We are passing cb_server_fd from output param in case OO
		 * is a CBObj. For CBObj, we have to ensure that it is sent
		 * to server who serves it and that info comes from USpace.
		 */
		ret = get_uhandle_from_tzhandle(tz_args->handle,
					TZHANDLE_GET_SERVER(tz_args->handle),
				(int32_t *)&(args_buf[i].o.fd), NO_LOCK);
		if (ret)
			goto out;
		tz_args++;
	}
	ret = 0;
out:
	return ret;
}

static bool is_inbound_req(int val)
{
	return (val == SMCINVOKE_RESULT_INBOUND_REQ_NEEDED ||
		val == QSEOS_RESULT_INCOMPLETE ||
		val == QSEOS_RESULT_BLOCKED_ON_LISTENER);
}

static int prepare_send_scm_msg(const uint8_t *in_buf, phys_addr_t in_paddr,
				size_t in_buf_len,
				uint8_t *out_buf, phys_addr_t out_paddr,
				size_t out_buf_len,
				struct smcinvoke_cmd_req *req,
				union smcinvoke_arg *args_buf,
				bool *tz_acked)
{
	int ret = 0, cmd;
	u64 response_type;
	unsigned int data;
	struct file *arr_filp[OBJECT_COUNTS_MAX_OO] = {NULL};

	*tz_acked = false;
	/* buf size should be page aligned */
	if ((in_buf_len % PAGE_SIZE) != 0 || (out_buf_len % PAGE_SIZE) != 0)
		return -EINVAL;

	cmd = SMCINVOKE_INVOKE_CMD;
	/*
	 * purpose of lock here is to ensure that any CB obj that may be going
	 * to user as OO is not released by piggyback message on another invoke
	 * request. We should not move this lock to process_invoke_req() because
	 * that will either cause deadlock or prevent any other invoke request
	 * to come in. We release this lock when either
	 *     a) TZ requires HLOS action to complete ongoing invoke operation
	 *     b) Final response to invoke has been marshalled out
	 */
	while (1) {
		mutex_lock(&g_smcinvoke_lock);

		if (cmd == SMCINVOKE_INVOKE_CMD)
			ret = qcom_scm_invoke_smc(in_paddr, in_buf_len,
						out_paddr, out_buf_len,
						&req->result, &response_type,
						&data);
		else
			ret = qcom_scm_invoke_callback_response(
					virt_to_phys(out_buf), out_buf_len,
					&req->result, &response_type, &data);

		if (!ret && !is_inbound_req(response_type)) {
			/* dont marshal if Obj returns an error */
			if (!req->result) {
				if (args_buf != NULL)
					ret = marshal_out_invoke_req(in_buf,
						in_buf_len, req, args_buf);
			}
			*tz_acked = true;
		}
		mutex_unlock(&g_smcinvoke_lock);

		if (cmd == SMCINVOKE_CB_RSP_CMD)
			release_filp(arr_filp, OBJECT_COUNTS_MAX_OO);

		if (ret || !is_inbound_req(response_type))
			break;

		/* process listener request */
		if (response_type == QSEOS_RESULT_INCOMPLETE ||
		    response_type == QSEOS_RESULT_BLOCKED_ON_LISTENER) {
			ret = qseecom_process_listener_from_smcinvoke(
					&req->result, &response_type, &data);
			/*
			 * new scm APIs do not provide complete response i.e. res[0-2],
			 * we loose some values returned from QSEECom APIs. so we need to
			 * populate result from response type i.e. res[1]
			 */
			req->result = response_type;
			if (!req->result) {
				ret = marshal_out_invoke_req(in_buf,
						in_buf_len, req, args_buf);
			}
			*tz_acked = true;
		}

		/*
		 * qseecom does not understand smcinvoke's callback object &&
		 * erringly sets ret value as -EINVAL :( We need to handle it.
		 */
		if (response_type != SMCINVOKE_RESULT_INBOUND_REQ_NEEDED)
			break;

		if (response_type == SMCINVOKE_RESULT_INBOUND_REQ_NEEDED) {
			pr_debug("looks like inbnd req reqd\n");
			process_tzcb_req(out_buf, out_buf_len, arr_filp);
			cmd = SMCINVOKE_CB_RSP_CMD;
		}
	}
	return ret;
}
/*
 * SMC expects arguments in following format
 * ---------------------------------------------------------------------------
 * | cxt | op | counts | ptr|size |ptr|size...|ORef|ORef|...| rest of payload |
 * ---------------------------------------------------------------------------
 * cxt: target, op: operation, counts: total arguments
 * offset: offset is from beginning of buffer i.e. cxt
 * size: size is 8 bytes aligned value
 */
static size_t compute_in_msg_size(const struct smcinvoke_cmd_req *req,
					const union smcinvoke_arg *args_buf)
{
	uint32_t i = 0;

	size_t total_size = sizeof(struct smcinvoke_msg_hdr) +
				OBJECT_COUNTS_TOTAL(req->counts) *
					sizeof(union smcinvoke_tz_args);

	/* Computed total_size should be 8 bytes aligned from start of buf */
	total_size = ALIGN(total_size, SMCINVOKE_ARGS_ALIGN_SIZE);

	/* each buffer has to be 8 bytes aligned */
	while (i < OBJECT_COUNTS_NUM_buffers(req->counts))
		total_size = size_add(total_size,
		size_align(args_buf[i++].b.size, SMCINVOKE_ARGS_ALIGN_SIZE));

	return PAGE_ALIGN(total_size);
}

static int marshal_in_invoke_req(const struct smcinvoke_cmd_req *req,
			const union smcinvoke_arg *args_buf, uint32_t tzhandle,
			uint8_t *buf, size_t buf_size, struct file **arr_filp,
			int32_t *tzhandles_to_release)
{
	int ret = -EINVAL, i = 0, j = 0, k = 0;
	const struct smcinvoke_msg_hdr msg_hdr = {
					tzhandle, req->op, req->counts};
	uint32_t offset = sizeof(struct smcinvoke_msg_hdr) +
				sizeof(union smcinvoke_tz_args) *
				OBJECT_COUNTS_TOTAL(req->counts);
	union smcinvoke_tz_args *tz_args = NULL;

	if (buf_size < offset)
		goto out;

	*(struct smcinvoke_msg_hdr *)buf = msg_hdr;
	tz_args = (union smcinvoke_tz_args *)(buf +
				sizeof(struct smcinvoke_msg_hdr));

	if (args_buf == NULL)
		return 0;

	FOR_ARGS(i, req->counts, BI) {
		offset = size_align(offset, SMCINVOKE_ARGS_ALIGN_SIZE);
		if ((offset > buf_size) ||
			(args_buf[i].b.size > (buf_size - offset)))
			goto out;

		tz_args[i].b.offset = offset;
		tz_args[i].b.size = args_buf[i].b.size;

		if (copy_from_user(buf + offset,
			(void __user *)(uintptr_t)(args_buf[i].b.addr),
						args_buf[i].b.size))
			goto out;

		offset += args_buf[i].b.size;
	}
	FOR_ARGS(i, req->counts, BO) {
		offset = size_align(offset, SMCINVOKE_ARGS_ALIGN_SIZE);
		if ((offset > buf_size) ||
			(args_buf[i].b.size > (buf_size - offset)))
			goto out;

		tz_args[i].b.offset = offset;
		tz_args[i].b.size = args_buf[i].b.size;
		offset += args_buf[i].b.size;
	}
	FOR_ARGS(i, req->counts, OI) {
		ret = get_tzhandle_from_uhandle(args_buf[i].o.fd,
				 args_buf[i].o.cb_server_fd, &arr_filp[j++],
							&(tz_args[i].handle));
		if (ret)
			goto out;
		tzhandles_to_release[k++] = tz_args[i].handle;
	}
	ret = 0;
out:
	return ret;
}

static int marshal_in_tzcb_req(const struct smcinvoke_cb_txn *cb_txn,
				struct smcinvoke_accept *user_req, int srvr_id)
{
	int ret = 0, i = 0;
	union smcinvoke_arg tmp_arg;
	struct smcinvoke_tzcb_req *tzcb_req = cb_txn->cb_req;
	union smcinvoke_tz_args *tz_args = tzcb_req->args;
	size_t tzcb_req_len = cb_txn->cb_req_bytes;
	size_t tz_buf_offset = TZCB_BUF_OFFSET(tzcb_req);
	size_t user_req_buf_offset = sizeof(union smcinvoke_arg) *
				OBJECT_COUNTS_TOTAL(tzcb_req->hdr.counts);

	if (tz_buf_offset > tzcb_req_len) {
		ret = -EINVAL;
		goto out;
	}

	user_req->txn_id = cb_txn->txn_id;
	if (get_uhandle_from_tzhandle(tzcb_req->hdr.tzhandle, srvr_id,
				&user_req->cbobj_id, TAKE_LOCK)) {
		ret = -EINVAL;
		goto out;
	}
	user_req->op = tzcb_req->hdr.op;
	user_req->counts = tzcb_req->hdr.counts;
	user_req->argsize = sizeof(union smcinvoke_arg);

	FOR_ARGS(i, tzcb_req->hdr.counts, BI) {
		user_req_buf_offset = size_align(user_req_buf_offset,
					 SMCINVOKE_ARGS_ALIGN_SIZE);
		tmp_arg.b.size = tz_args[i].b.size;
		if ((tz_args[i].b.offset > tzcb_req_len) ||
		    (tz_args[i].b.size > tzcb_req_len - tz_args[i].b.offset) ||
		    (user_req_buf_offset > user_req->buf_len) ||
			(tmp_arg.b.size >
			user_req->buf_len - user_req_buf_offset)) {
			ret = -EINVAL;
			pr_err("%s: buffer overflow detected\n", __func__);
			goto out;
		}
		tmp_arg.b.addr = user_req->buf_addr + user_req_buf_offset;

		if (copy_to_user(u64_to_user_ptr
				(user_req->buf_addr + i * sizeof(tmp_arg)),
				&tmp_arg, sizeof(tmp_arg)) ||
		   copy_to_user(u64_to_user_ptr(tmp_arg.b.addr),
			(uint8_t *)(tzcb_req) + tz_args[i].b.offset,
				tz_args[i].b.size)) {
			ret = -EFAULT;
			goto out;
		}
		user_req_buf_offset += tmp_arg.b.size;
	}
	FOR_ARGS(i, tzcb_req->hdr.counts, BO) {
		user_req_buf_offset = size_align(user_req_buf_offset,
					SMCINVOKE_ARGS_ALIGN_SIZE);

		tmp_arg.b.size = tz_args[i].b.size;
		if ((user_req_buf_offset > user_req->buf_len) ||
		    (tmp_arg.b.size >
				user_req->buf_len - user_req_buf_offset)) {
			ret = -EINVAL;
			pr_err("%s: buffer overflow detected\n", __func__);
			goto out;
		}
		tmp_arg.b.addr = user_req->buf_addr + user_req_buf_offset;

		if (copy_to_user(u64_to_user_ptr
				(user_req->buf_addr + i * sizeof(tmp_arg)),
				&tmp_arg, sizeof(tmp_arg))) {
			ret = -EFAULT;
			goto out;
		}
		user_req_buf_offset += tmp_arg.b.size;
	}
	FOR_ARGS(i, tzcb_req->hdr.counts, OI) {
		/*
		 * create a new FD and assign to output object's
		 * context
		 */
		ret = get_uhandle_from_tzhandle(tz_args[i].handle, srvr_id,
					(int32_t *)&(tmp_arg.o.fd), TAKE_LOCK);
		if (ret) {
			ret = -EINVAL;
			goto out;
		}
		if (copy_to_user(u64_to_user_ptr
				(user_req->buf_addr + i * sizeof(tmp_arg)),
				&tmp_arg, sizeof(tmp_arg))) {
			ret = -EFAULT;
			goto out;
		}
	}
out:
	return ret;
}

static int marshal_out_tzcb_req(const struct smcinvoke_accept *user_req,
				struct smcinvoke_cb_txn *cb_txn,
				struct file **arr_filp)
{
	int ret = -EINVAL, i = 0;
	int32_t tzhandles_to_release[OBJECT_COUNTS_MAX_OO] = {0};
	struct smcinvoke_tzcb_req *tzcb_req = cb_txn->cb_req;
	union smcinvoke_tz_args *tz_args = tzcb_req->args;

	release_tzhandles(&cb_txn->cb_req->hdr.tzhandle, 1);
	tzcb_req->result = user_req->result;
	FOR_ARGS(i, tzcb_req->hdr.counts, BO) {
		union smcinvoke_arg tmp_arg;

		if (copy_from_user((uint8_t *)&tmp_arg, u64_to_user_ptr(
			user_req->buf_addr + i * sizeof(union smcinvoke_arg)),
			sizeof(union smcinvoke_arg))) {
			ret = -EFAULT;
			goto out;
		}
		if (tmp_arg.b.size > tz_args[i].b.size)
			goto out;
		if (copy_from_user((uint8_t *)(tzcb_req) + tz_args[i].b.offset,
					u64_to_user_ptr(tmp_arg.b.addr),
						tmp_arg.b.size)) {
			ret = -EFAULT;
			goto out;
		}
	}

	FOR_ARGS(i, tzcb_req->hdr.counts, OO) {
		union smcinvoke_arg tmp_arg;

		if (copy_from_user((uint8_t *)&tmp_arg, u64_to_user_ptr(
			user_req->buf_addr + i * sizeof(union smcinvoke_arg)),
			sizeof(union smcinvoke_arg))) {
			ret = -EFAULT;
			goto out;
		}
		ret = get_tzhandle_from_uhandle(tmp_arg.o.fd,
				tmp_arg.o.cb_server_fd, &arr_filp[i],
						&(tz_args[i].handle));
		if (ret)
			goto out;
		tzhandles_to_release[i] = tz_args[i].handle;
	}
	FOR_ARGS(i, tzcb_req->hdr.counts, OI) {
		if (TZHANDLE_IS_CB_OBJ(tz_args[i].handle))
			release_tzhandles(&tz_args[i].handle, 1);
	}
	ret = 0;
out:
	if (ret)
		release_tzhandles(tzhandles_to_release, OBJECT_COUNTS_MAX_OO);
	return ret;
}

static void process_piggyback_data(void *buf, size_t buf_size)
{
	int i;
	struct smcinvoke_tzcb_req req = {0};
	struct smcinvoke_piggyback_msg *msg = buf;
	int32_t *objs = msg->objs;

	for (i = 0; i < msg->counts; i++) {
		req.hdr.op = msg->op;
		req.hdr.counts = 0; /* release op does not require any args */
		req.hdr.tzhandle = objs[i];
		process_tzcb_req(&req, sizeof(struct smcinvoke_tzcb_req), NULL);
		/* cbobjs_in_flight will be adjusted during CB processing */
	}
}


static long process_ack_local_obj(struct file *filp, unsigned int cmd,
						unsigned long arg)
{
	int ret = -1;
	int32_t local_obj = SMCINVOKE_USERSPACE_OBJ_NULL;
	struct smcinvoke_file_data *filp_data = filp->private_data;

	if (_IOC_SIZE(cmd) != sizeof(int32_t))
		return -EINVAL;

	ret = copy_from_user(&local_obj, (void __user *)(uintptr_t)arg,
					sizeof(int32_t));
	if (ret)
		return -EFAULT;

	mutex_lock(&g_smcinvoke_lock);
	if (UHANDLE_IS_CB_OBJ(local_obj))
		ret = put_pending_cbobj_locked(filp_data->server_id,
					UHANDLE_GET_CB_OBJ(local_obj));
	mutex_unlock(&g_smcinvoke_lock);

	return ret;
}

static long process_server_req(struct file *filp, unsigned int cmd,
						 unsigned long arg)
{
	int ret = -1;
	int32_t server_fd = -1;
	struct smcinvoke_server server_req = {0};
	struct smcinvoke_server_info *server_info = NULL;

	if (_IOC_SIZE(cmd) != sizeof(server_req)) {
		pr_err("invalid command size received for server request\n");
		return -EINVAL;
	}
	ret = copy_from_user(&server_req, (void __user *)(uintptr_t)arg,
					sizeof(server_req));
	if (ret) {
		pr_err("copying server request from user failed\n");
		return -EFAULT;
	}
	server_info = kzalloc(sizeof(*server_info), GFP_KERNEL);
	if (!server_info)
		return -ENOMEM;

	kref_init(&server_info->ref_cnt);
	init_waitqueue_head(&server_info->req_wait_q);
	init_waitqueue_head(&server_info->rsp_wait_q);
	server_info->cb_buf_size = server_req.cb_buf_size;
	hash_init(server_info->reqs_table);
	hash_init(server_info->responses_table);
	INIT_LIST_HEAD(&server_info->pending_cbobjs);

	mutex_lock(&g_smcinvoke_lock);

	server_info->server_id = next_cb_server_id_locked();
	hash_add(g_cb_servers, &server_info->hash,
					server_info->server_id);
	if (g_max_cb_buf_size < server_req.cb_buf_size)
		g_max_cb_buf_size = server_req.cb_buf_size;

	mutex_unlock(&g_smcinvoke_lock);
	ret = get_fd_for_obj(SMCINVOKE_OBJ_TYPE_SERVER,
				server_info->server_id, &server_fd);

	if (ret)
		release_cb_server(server_info->server_id);

	return server_fd;
}

static long process_accept_req(struct file *filp, unsigned int cmd,
						unsigned long arg)
{
	int ret = -1;
	struct smcinvoke_file_data *server_obj = filp->private_data;
	struct smcinvoke_accept user_args = {0};
	struct smcinvoke_cb_txn *cb_txn = NULL;
	struct smcinvoke_server_info *server_info = NULL;

	if (_IOC_SIZE(cmd) != sizeof(struct smcinvoke_accept)) {
		pr_err("command size invalid for accept request\n");
		return -EINVAL;
	}

	if (copy_from_user(&user_args, (void __user *)arg,
					sizeof(struct smcinvoke_accept))) {
		pr_err("copying accept request from user failed\n");
		return -EFAULT;
	}

	if (user_args.argsize != sizeof(union smcinvoke_arg)) {
		pr_err("arguments size is invalid for accept thread\n");
		return -EINVAL;
	}

	/* ACCEPT is available only on server obj */
	if (server_obj->context_type != SMCINVOKE_OBJ_TYPE_SERVER) {
		pr_err("invalid object type received for accept req\n");
		return -EPERM;
	}

	mutex_lock(&g_smcinvoke_lock);
	server_info = get_cb_server_locked(server_obj->server_id);

	if (!server_info) {
		pr_err("No matching server with server id : %u found\n",
					server_obj->server_id);
		mutex_unlock(&g_smcinvoke_lock);
		return -EINVAL;
	}

	if (server_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT)
		server_info->state = 0;

	mutex_unlock(&g_smcinvoke_lock);

	/* First check if it has response otherwise wait for req */
	if (user_args.has_resp) {
		mutex_lock(&g_smcinvoke_lock);
		cb_txn = find_cbtxn_locked(server_info, user_args.txn_id,
					SMCINVOKE_REQ_PROCESSING);
		mutex_unlock(&g_smcinvoke_lock);
		/*
		 * cb_txn can be null if userspace provides wrong txn id OR
		 * invoke thread died while server was processing cb req.
		 * if invoke thread dies, it would remove req from Q. So
		 * no matching cb_txn would be on Q and hence NULL cb_txn.
		 * In this case, we want this thread to come back and start
		 * waiting for new cb requests, hence return EAGAIN here
		 */
		if (!cb_txn) {
			pr_err("%s txn %d either invalid or removed from Q\n",
					__func__, user_args.txn_id);
			ret = -EAGAIN;
			goto out;
		}
		ret = marshal_out_tzcb_req(&user_args, cb_txn,
						cb_txn->filp_to_release);
		/*
		 * if client did not set error and we get error locally,
		 * we return local error to TA
		 */
		if (ret && cb_txn->cb_req->result == 0)
			cb_txn->cb_req->result = OBJECT_ERROR_UNAVAIL;

		cb_txn->state = SMCINVOKE_REQ_PROCESSED;
		kref_put(&cb_txn->ref_cnt, delete_cb_txn);
		wake_up(&server_info->rsp_wait_q);
		/*
		 * if marshal_out fails, we should let userspace release
		 * any ref/obj it created for CB processing
		 */
		if (ret && OBJECT_COUNTS_NUM_OO(user_args.counts))
			goto out;
	}
	/*
	 * Once response has been delivered, thread will wait for another
	 * callback req to process.
	 */
	do {
		ret = wait_event_interruptible(server_info->req_wait_q,
				!hash_empty(server_info->reqs_table));
		if (ret) {
			pr_debug("%s wait_event interrupted: ret = %d\n",
							__func__, ret);
			/*
			 * Ideally, we should destroy server if accept threads
			 * are returning due to client being killed or device
			 * going down (Shutdown/Reboot) but that would make
			 * server_info invalid. Other accept/invoke threads are
			 * using server_info and would crash. So dont do that.
			 */
			mutex_lock(&g_smcinvoke_lock);
			server_info->state = SMCINVOKE_SERVER_STATE_DEFUNCT;
			mutex_unlock(&g_smcinvoke_lock);
			wake_up_interruptible(&server_info->rsp_wait_q);
			goto out;
		}
		mutex_lock(&g_smcinvoke_lock);
		cb_txn = find_cbtxn_locked(server_info,
						SMCINVOKE_NEXT_AVAILABLE_TXN,
						SMCINVOKE_REQ_PLACED);
		mutex_unlock(&g_smcinvoke_lock);
		if (cb_txn) {
			cb_txn->state = SMCINVOKE_REQ_PROCESSING;
			ret = marshal_in_tzcb_req(cb_txn, &user_args,
							server_obj->server_id);
			if (ret) {
				pr_err("failed to marshal in the callback request\n");
				cb_txn->cb_req->result = OBJECT_ERROR_UNAVAIL;
				cb_txn->state = SMCINVOKE_REQ_PROCESSED;
				kref_put(&cb_txn->ref_cnt, delete_cb_txn);
				wake_up_interruptible(&server_info->rsp_wait_q);
				continue;
			}
			mutex_lock(&g_smcinvoke_lock);
			hash_add(server_info->responses_table, &cb_txn->hash,
							cb_txn->txn_id);
			kref_put(&cb_txn->ref_cnt, delete_cb_txn);
			mutex_unlock(&g_smcinvoke_lock);
			ret =  copy_to_user((void __user *)arg, &user_args,
					sizeof(struct smcinvoke_accept));
		}
	} while (!cb_txn);
out:
	if (server_info)
		kref_put(&server_info->ref_cnt, destroy_cb_server);

	if (ret && ret != -ERESTARTSYS)
		pr_err("accept thread returning with ret: %d\n", ret);

	return ret;
}

static long process_invoke_req(struct file *filp, unsigned int cmd,
						unsigned long arg)
{
	int    ret = -1, nr_args = 0;
	struct smcinvoke_cmd_req req = {0};
	void   *in_msg = NULL, *out_msg = NULL;
	size_t inmsg_size = 0, outmsg_size = SMCINVOKE_TZ_MIN_BUF_SIZE;
	union  smcinvoke_arg *args_buf = NULL;
	struct smcinvoke_file_data *tzobj = filp->private_data;
	struct qtee_shm in_shm = {0}, out_shm = {0};

	/*
	 * Hold reference to remote object until invoke op is not
	 * completed. Release once invoke is done.
	 */
	struct file *filp_to_release[OBJECT_COUNTS_MAX_OO] = {NULL};
	/*
	 * If anything goes wrong, release alloted tzhandles for
	 * local objs which could be either CBObj or MemObj.
	 */
	int32_t tzhandles_to_release[OBJECT_COUNTS_MAX_OO] = {0};
	bool tz_acked = false;

	if (_IOC_SIZE(cmd) != sizeof(req)) {
		pr_err("command size for invoke req is invalid\n");
		return -EINVAL;
	}

	if (tzobj->context_type != SMCINVOKE_OBJ_TYPE_TZ_OBJ) {
		pr_err("object type for invoke req is invalid\n");
		return -EPERM;
	}

	ret = copy_from_user(&req, (void __user *)arg, sizeof(req));
	if (ret) {
		pr_err("copying invoke req failed\n");
		return -EFAULT;
	}

	if (req.argsize != sizeof(union smcinvoke_arg)) {
		pr_err("arguments size for invoke req is invalid\n");
		return -EINVAL;
	}

	nr_args = OBJECT_COUNTS_NUM_buffers(req.counts) +
			OBJECT_COUNTS_NUM_objects(req.counts);

	if (nr_args) {
		args_buf = kcalloc(nr_args, req.argsize, GFP_KERNEL);
		if (!args_buf)
			return -ENOMEM;

		ret = copy_from_user(args_buf, u64_to_user_ptr(req.args),
					nr_args * req.argsize);

		if (ret) {
			ret = -EFAULT;
			goto out;
		}
	}

	inmsg_size = compute_in_msg_size(&req, args_buf);
	ret = qtee_shmbridge_allocate_shm(inmsg_size, &in_shm);
	if (ret) {
		ret = -ENOMEM;
		pr_err("shmbridge alloc failed for in msg in invoke req\n");
		goto out;
	}
	in_msg = in_shm.vaddr;

	mutex_lock(&g_smcinvoke_lock);
	outmsg_size = PAGE_ALIGN(g_max_cb_buf_size);
	mutex_unlock(&g_smcinvoke_lock);
	ret = qtee_shmbridge_allocate_shm(outmsg_size, &out_shm);
	if (ret) {
		ret = -ENOMEM;
		pr_err("shmbridge alloc failed for out msg in invoke req\n");
		goto out;
	}
	out_msg = out_shm.vaddr;

	ret = marshal_in_invoke_req(&req, args_buf, tzobj->tzhandle, in_msg,
			inmsg_size, filp_to_release, tzhandles_to_release);
	if (ret) {
		pr_err("failed to marshal in invoke req, ret :%d\n", ret);
		goto out;
	}

	ret = prepare_send_scm_msg(in_msg, in_shm.paddr, inmsg_size,
					out_msg, out_shm.paddr, outmsg_size,
					&req, args_buf, &tz_acked);

	/*
	 * If scm_call is success, TZ owns responsibility to release
	 * refs for local objs.
	 */
	if (!tz_acked) {
		pr_debug("scm call successful\n");
		goto out;
	}
	memset(tzhandles_to_release, 0, sizeof(tzhandles_to_release));

	/*
	 * if invoke op results in an err, no need to marshal_out and
	 * copy args buf to user space
	 */
	if (!req.result) {
		/*
		 * Dont check ret of marshal_out because there might be a
		 * FD for OO which userspace must release even if an error
		 * occurs. Releasing FD from user space is much simpler than
		 * doing here. ORing of ret is reqd not to miss past error
		 */
		ret |=  copy_to_user(u64_to_user_ptr(req.args), args_buf,
					nr_args * req.argsize);
	}
	/* copy result of invoke op */
	ret |=  copy_to_user((void __user *)arg, &req, sizeof(req));
	if (ret)
		goto out;

	/* Outbuf could be carrying local objs to be released. */
	process_piggyback_data(out_msg, outmsg_size);
out:
	release_filp(filp_to_release, OBJECT_COUNTS_MAX_OO);
	if (ret)
		release_tzhandles(tzhandles_to_release, OBJECT_COUNTS_MAX_OO);
	qtee_shmbridge_free_shm(&in_shm);
	qtee_shmbridge_free_shm(&out_shm);
	kfree(args_buf);

	if (ret)
		pr_err("invoke thread returning with ret = %d\n", ret);

	return ret;
}

static long smcinvoke_ioctl(struct file *filp, unsigned int cmd,
						unsigned long arg)
{
	long ret = 0;

	switch (cmd) {
	case SMCINVOKE_IOCTL_INVOKE_REQ:
		ret = process_invoke_req(filp, cmd, arg);
		break;
	case SMCINVOKE_IOCTL_ACCEPT_REQ:
		ret = process_accept_req(filp, cmd, arg);
		break;
	case SMCINVOKE_IOCTL_SERVER_REQ:
		ret = process_server_req(filp, cmd, arg);
		break;
	case SMCINVOKE_IOCTL_ACK_LOCAL_OBJ:
		ret = process_ack_local_obj(filp, cmd, arg);
		break;
	default:
		ret = -ENOIOCTLCMD;
		break;
	}
	return ret;
}

static int smcinvoke_open(struct inode *nodp, struct file *filp)
{
	struct smcinvoke_file_data *tzcxt = NULL;

	tzcxt = kzalloc(sizeof(*tzcxt), GFP_KERNEL);
	if (!tzcxt)
		return -ENOMEM;

	tzcxt->tzhandle = SMCINVOKE_TZ_ROOT_OBJ;
	tzcxt->context_type = SMCINVOKE_OBJ_TYPE_TZ_OBJ;
	filp->private_data = tzcxt;

	return 0;
}

static int release_cb_server(uint16_t server_id)
{
	struct smcinvoke_server_info *server = NULL;

	mutex_lock(&g_smcinvoke_lock);
	server = find_cb_server_locked(server_id);
	if (server)
		kref_put(&server->ref_cnt, destroy_cb_server);
	mutex_unlock(&g_smcinvoke_lock);
	return 0;
}

static int smcinvoke_release(struct inode *nodp, struct file *filp)
{
	int ret = 0;
	bool release_handles;
	uint8_t *in_buf = NULL;
	uint8_t *out_buf = NULL;
	struct smcinvoke_msg_hdr hdr = {0};
	struct smcinvoke_file_data *file_data = filp->private_data;
	struct smcinvoke_cmd_req req = {0};
	uint32_t tzhandle = 0;
	struct qtee_shm in_shm = {0}, out_shm = {0};

	if (file_data->context_type == SMCINVOKE_OBJ_TYPE_SERVER) {
		ret = release_cb_server(file_data->server_id);
		goto out;
	}

	tzhandle = file_data->tzhandle;
	/* Root object is special in sense it is indestructible */
	if (!tzhandle || tzhandle == SMCINVOKE_TZ_ROOT_OBJ)
		goto out;

	ret = qtee_shmbridge_allocate_shm(SMCINVOKE_TZ_MIN_BUF_SIZE, &in_shm);
	if (ret) {
		ret = -ENOMEM;
		pr_err("shmbridge alloc failed for in msg in release\n");
		goto out;
	}

	ret = qtee_shmbridge_allocate_shm(SMCINVOKE_TZ_MIN_BUF_SIZE, &out_shm);
	if (ret) {
		ret = -ENOMEM;
		pr_err("shmbridge alloc failed for out msg in release\n");
		goto out;
	}

	in_buf = in_shm.vaddr;
	out_buf = out_shm.vaddr;
	hdr.tzhandle = tzhandle;
	hdr.op = OBJECT_OP_RELEASE;
	hdr.counts = 0;
	*(struct smcinvoke_msg_hdr *)in_buf = hdr;

	ret = prepare_send_scm_msg(in_buf, in_shm.paddr,
		SMCINVOKE_TZ_MIN_BUF_SIZE, out_buf, out_shm.paddr,
		SMCINVOKE_TZ_MIN_BUF_SIZE, &req, NULL, &release_handles);

	process_piggyback_data(out_buf, SMCINVOKE_TZ_MIN_BUF_SIZE);
out:
	kfree(filp->private_data);
	qtee_shmbridge_free_shm(&in_shm);
	qtee_shmbridge_free_shm(&out_shm);

	return ret;
}

static int smcinvoke_probe(struct platform_device *pdev)
{
	unsigned int baseminor = 0;
	unsigned int count = 1;
	int rc = 0;

	rc = alloc_chrdev_region(&smcinvoke_device_no, baseminor, count,
							SMCINVOKE_DEV);
	if (rc < 0) {
		pr_err("chrdev_region failed %d for %s\n", rc, SMCINVOKE_DEV);
		return rc;
	}
	driver_class = class_create(THIS_MODULE, SMCINVOKE_DEV);
	if (IS_ERR(driver_class)) {
		rc = -ENOMEM;
		pr_err("class_create failed %d\n", rc);
		goto exit_unreg_chrdev_region;
	}
	class_dev = device_create(driver_class, NULL, smcinvoke_device_no,
						NULL, SMCINVOKE_DEV);
	if (!class_dev) {
		pr_err("class_device_create failed %d\n", rc);
		rc = -ENOMEM;
		goto exit_destroy_class;
	}

	cdev_init(&smcinvoke_cdev, &g_smcinvoke_fops);
	smcinvoke_cdev.owner = THIS_MODULE;

	rc = cdev_add(&smcinvoke_cdev, MKDEV(MAJOR(smcinvoke_device_no), 0),
								count);
	if (rc < 0) {
		pr_err("cdev_add failed %d for %s\n", rc, SMCINVOKE_DEV);
		goto exit_destroy_device;
	}
	smcinvoke_pdev = pdev;
	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
	if (rc) {
		pr_err("dma_set_mask_and_coherent failed %d\n", rc);
		goto exit_destroy_device;
	}

	return  0;

exit_destroy_device:
	device_destroy(driver_class, smcinvoke_device_no);
exit_destroy_class:
	class_destroy(driver_class);
exit_unreg_chrdev_region:
	unregister_chrdev_region(smcinvoke_device_no, count);
	return rc;
}

static int smcinvoke_remove(struct platform_device *pdev)
{
	int count = 1;

	cdev_del(&smcinvoke_cdev);
	device_destroy(driver_class, smcinvoke_device_no);
	class_destroy(driver_class);
	unregister_chrdev_region(smcinvoke_device_no, count);
	return 0;
}

static int __maybe_unused smcinvoke_suspend(struct platform_device *pdev,
					pm_message_t state)
{
	int ret = 0;

	mutex_lock(&g_smcinvoke_lock);
	if (cb_reqs_inflight) {
		pr_err("Failed to suspend smcinvoke driver\n");
		ret = -EIO;
	}
	mutex_unlock(&g_smcinvoke_lock);
	return ret;
}

static int __maybe_unused smcinvoke_resume(struct platform_device *pdev)
{
	return 0;
}

static const struct of_device_id smcinvoke_match[] = {
	{
		.compatible = "qcom,smcinvoke",
	},
	{},
};

static struct platform_driver smcinvoke_plat_driver = {
	.probe = smcinvoke_probe,
	.remove = smcinvoke_remove,
	.suspend = smcinvoke_suspend,
	.resume = smcinvoke_resume,
	.driver = {
		.name = "smcinvoke",
		.of_match_table = smcinvoke_match,
	},
};

static int smcinvoke_init(void)
{
	return platform_driver_register(&smcinvoke_plat_driver);
}

static void smcinvoke_exit(void)
{
	platform_driver_unregister(&smcinvoke_plat_driver);
}

module_init(smcinvoke_init);
module_exit(smcinvoke_exit);

MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("SMC Invoke driver");